def stop(self, cancelled=False):
statsd.incr("timer.stopped")
if self.is_stopped():
raise ValueError(u"The timer has already been stopped")
last_log = self.log_set.order_by("created").last() # get last log
if not last_log and not cancelled:
raise ValueError(u"You can't stop a timer without a log")
# stop and update this model
self.stopped = postgres_now()
self.cancelled = cancelled
if last_log:
self.linked_case = last_log.case
self.save()
if self.linked_case:
# update billable time on case
cursor = connection.cursor()
cursor.execute(
"""
select sum(ceiling(EXTRACT(epoch FROM a.stopped-a.created)))
from timer_timer as a
where
a.cancelled = false and
a.stopped is not null and a.linked_case_id = %s""",
[self.linked_case.id],
)
total_billable_time, = cursor.fetchone()
if total_billable_time:
self.linked_case.billable_time = total_billable_time
if total_billable_time:
statsd.timing("timer.total_time", total_billable_time * 1000)
self.linked_case.save(update_fields=["billable_time"])
def stop(self, cancelled=False):
statsd.incr('timer.stopped')
if self.is_stopped():
raise ValueError(u'The timer has already been stopped')
last_log = self.log_set.order_by('created').last() # get last log
if not last_log and not cancelled:
raise ValueError(u'You can\'t stop a timer without a log')
# stop and update this model
self.stopped = timezone.now() # stop
self.cancelled = cancelled
if last_log:
self.linked_case = last_log.case
self.save()
if self.linked_case:
# update billable time on case
cursor = connection.cursor()
cursor.execute('''
select sum(ceiling(EXTRACT(epoch FROM a.stopped-a.created)))
from timer_timer as a
where
a.cancelled = false and
a.stopped is not null and a.linked_case_id = %s''', [self.linked_case.id])
total_billable_time, = cursor.fetchone()
if total_billable_time:
self.linked_case.billable_time = total_billable_time
if total_billable_time:
statsd.timing('timer.total_time', total_billable_time * 1000)
self.linked_case.save(update_fields=['billable_time'])
def trans_start_url(request):
"""
JSON handler to get the Bango payment URL to start a transaction.
"""
trans = None
trans_id = request.session.get('trans_id')
data = {'url': None, 'status': None}
if not trans_id:
log.error('trans_start_url(): no transaction ID in session')
return http.HttpResponseBadRequest()
try:
statsd.incr('purchase.payment_time.retry')
with statsd.timer('purchase.payment_time.get_transaction'):
trans = solitude.get_transaction(trans_id)
data['status'] = trans['status']
data['provider'] = constants.PROVIDERS_INVERTED[trans['provider']]
except ObjectDoesNotExist:
log.error('trans_start_url() transaction does not exist: {t}'
.format(t=trans_id))
if data['status'] == constants.STATUS_PENDING:
statsd.incr('purchase.payment_time.success')
payment_start = request.session.get('payment_start', False)
if payment_start:
delta = int((time.time() - float(payment_start)) * 1000)
statsd.timing('purchase.payment_time.duration', delta)
url = get_payment_url(trans)
log.info('async call got payment URL {url} for trans {tr}'
.format(url=url, tr=trans))
data['url'] = url
return data
def listen(self):
"""Listens for incoming messages on the Redis queue"""
while 1:
val = None
try:
val = self.bot_bus.blpop('q', 1)
# Track q length
ql = self.bot_bus.llen('q')
statsd.gauge(".".join(["plugins", "q"]), ql)
if val:
_, val = val
LOG.debug('Recieved: %s', val)
line = Line(json.loads(val), self)
# Calculate the transport latency between go and the plugins.
delta = datetime.utcnow().replace(tzinfo=utc) - line._received
statsd.timing(".".join(["plugins", "latency"]),
delta.total_seconds() * 1000)
self.dispatch(line)
except Exception:
LOG.error("Line Dispatch Failed", exc_info=True, extra={
"line": val
})
def _rebuild_kb_chunk(data):
"""Re-render a chunk of documents.
Note: Don't use host components when making redirects to wiki pages; those
redirects won't be auto-pruned when they're 404s.
"""
log.info('Rebuilding %s documents.' % len(data))
pin_this_thread() # Stick to master.
messages = []
start = time.time()
for pk in data:
message = None
try:
document = Document.objects.get(pk=pk)
# If we know a redirect link to be broken (i.e. if it looks like a
# link to a document but the document isn't there), log an error:
url = document.redirect_url()
if (url and points_to_document_view(url) and
not document.redirect_document()):
log.warn('Invalid redirect document: %d' % pk)
html = document.parse_and_calculate_links()
if document.html != html:
# We are calling update here to so we only update the html
# column instead of all of them. This bypasses post_save
# signal handlers like the one that triggers reindexing.
# See bug 797038 and bug 797352.
Document.objects.filter(pk=pk).update(html=html)
statsd.incr('wiki.rebuild_chunk.change')
else:
statsd.incr('wiki.rebuild_chunk.nochange')
except Document.DoesNotExist:
message = 'Missing document: %d' % pk
except Revision.DoesNotExist:
message = 'Missing revision for document: %d' % pk
except ValidationError as e:
message = 'ValidationError for %d: %s' % (pk, e.messages[0])
except SlugCollision:
message = 'SlugCollision: %d' % pk
except TitleCollision:
message = 'TitleCollision: %d' % pk
if message:
log.debug(message)
messages.append(message)
d = time.time() - start
statsd.timing('wiki.rebuild_chunk', int(round(d * 1000)))
if messages:
subject = ('[%s] Exceptions raised in _rebuild_kb_chunk()' %
settings.PLATFORM_NAME)
mail_admins(subject=subject, message='\n'.join(messages))
if not transaction.get_connection().in_atomic_block:
transaction.commit()
unpin_this_thread() # Not all tasks need to do use the master.
def get_detail(self, request, **kwargs):
"""
Returns a single serialized resource.
Calls ``cached_obj_get/obj_get`` to provide the data, then handles that result
set and serializes it.
Should return a HttpResponse (200 OK).
"""
basic_bundle = self.build_bundle(request=request)
try:
obj = self.cached_obj_get(bundle=basic_bundle, **self.remove_api_resource_names(kwargs))
except ObjectDoesNotExist:
return http.HttpNotFound()
except MultipleObjectsReturned:
return http.HttpMultipleChoices("More than one resource is found at this URI.")
bundle = self.build_bundle(obj=obj, request=request)
bundle = self.full_dehydrate(bundle)
#inserted here to display the datapoints for this particular dataset
try:
bundle.request.GET["no_points"]
except KeyError:
bundle = self.dehydrant_detail(bundle)
###
bundle = self.alter_detail_data_to_serialize(request, bundle)
statsd.timing("dataset_detail_read_time", int((time.time()-bundle.request._start_time)*1000))
return self.create_response(request, bundle)
def trans_start_url(request):
"""
JSON handler to get the Bango payment URL to start a transaction.
"""
try:
statsd.incr('purchase.payment_time.retry')
with statsd.timer('purchase.payment_time.get_transaction'):
trans = solitude.get_transaction(request.session['trans_id'])
except ObjectDoesNotExist:
log.error('trans_start_url() transaction does not exist: {t}'
.format(t=request.session['trans_id']))
trans = {'status': None}
data = {'url': None, 'status': trans['status']}
if trans['status'] == constants.STATUS_PENDING:
statsd.incr('purchase.payment_time.success')
payment_start = request.session.get('payment_start', False)
if payment_start:
delta = int((time.time() - float(payment_start)) * 1000)
statsd.timing('purchase.payment_time.duration', delta)
url = get_payment_url(trans)
log.info('async call got payment URL {url} for trans {tr}'
.format(url=url, tr=trans))
data['url'] = url
return data
def from_upload(cls, upload, addon, platforms, send_signal=True,
source=None, is_beta=False):
from olympia.addons.models import AddonFeatureCompatibility
data = utils.parse_addon(upload, addon)
try:
license = addon.versions.latest().license_id
except Version.DoesNotExist:
license = None
v = cls.objects.create(
addon=addon,
version=data['version'],
license_id=license,
source=source
)
log.info('New version: %r (%s) from %r' % (v, v.id, upload))
# Update the add-on e10s compatibility since we're creating a new
# version that may change that.
e10s_compatibility = data.get('e10s_compatibility')
if e10s_compatibility is not None:
feature_compatibility = (
AddonFeatureCompatibility.objects.get_or_create(addon=addon)[0]
)
feature_compatibility.update(e10s=e10s_compatibility)
AV = ApplicationsVersions
for app in data.get('apps', []):
AV(version=v, min=app.min, max=app.max,
application=app.id).save()
if addon.type == amo.ADDON_SEARCH:
# Search extensions are always for all platforms.
platforms = [amo.PLATFORM_ALL.id]
else:
platforms = cls._make_safe_platform_files(platforms)
for platform in platforms:
File.from_upload(upload, v, platform, parse_data=data,
is_beta=is_beta)
v.disable_old_files()
# After the upload has been copied to all platforms, remove the upload.
storage.delete(upload.path)
if send_signal:
version_uploaded.send(sender=v)
# Track the time it took from first upload through validation
# (and whatever else) until a version was created.
upload_start = utc_millesecs_from_epoch(upload.created)
now = datetime.datetime.now()
now_ts = utc_millesecs_from_epoch(now)
upload_time = now_ts - upload_start
log.info('Time for version {version} creation from upload: {delta}; '
'created={created}; now={now}'
.format(delta=upload_time, version=v,
created=upload.created, now=now))
statsd.timing('devhub.version_created_from_upload', upload_time)
return v
def time_status_change(sender, **kwargs):
# There's no status change if the transaction was just created.
if kwargs.get('raw', False) or kwargs.get('created', False):
return
obj = kwargs['instance']
status = constants.STATUSES_INVERTED[obj.status]
statsd.timing('transaction.status.{0}'.format(status),
(obj.modified - obj.created).seconds)
def process_key(start, key, value):
if 'timing' in key:
# Some values will be zero. We want the output of that to
# be zero relative to start.
value = max(start, int(value)) - start
statsd.timing(key, value)
elif key == 'window.performance.navigation.type':
statsd.incr('%s.%s' % (key, types[value]))
elif key == 'window.performance.navigation.redirectCount':
statsd.incr(key, int(value))
def time_f(fun, metric, *args, **kwargs):
start = time.time()
ret = fun(*args, **kwargs)
lapse = int((time.time() - start) * 1000)
if STATSD:
statsd.timing(metric, lapse)
else:
log = logging.getLogger(metric)
log.info('timing: %d', lapse)
return ret
def item_stats_report():
start = time.time()
d = get_item_counts_by_status()
statsd.gauge("items.total", d['total'])
statsd.gauge("items.open", d['open'])
statsd.gauge("items.inprogress", d['inprogress'])
statsd.gauge("items.resolved", d['resolved'])
statsd.gauge("items.closed", d['closed'])
statsd.gauge("items.verified", d['verified'])
end = time.time()
statsd.timing('celery.item_stats_report', int((end - start) * 1000))
def item_stats_report():
start = time.time()
d = get_item_counts_by_status()
statsd.gauge("items.total", d["total"])
statsd.gauge("items.open", d["open"])
statsd.gauge("items.inprogress", d["inprogress"])
statsd.gauge("items.resolved", d["resolved"])
statsd.gauge("items.closed", d["closed"])
statsd.gauge("items.verified", d["verified"])
end = time.time()
statsd.timing("celery.item_stats_report", int((end - start) * 1000))
def _process_summaries(start, keys):
calculated = {
"network": keys["window.performance.timing.responseStart"] - start,
"app": keys["window.performance.timing.domLoading"] - keys["window.performance.timing.responseStart"],
"dom": keys["window.performance.timing.domComplete"] - keys["window.performance.timing.domLoading"],
"rendering": keys["window.performance.timing.loadEventEnd"] - keys["window.performance.timing.domComplete"],
}
for k, v in calculated.items():
# If loadEventEnd still does not get populated, we could end up with
# negative numbers here.
statsd.timing("window.performance.calculated.%s" % k, max(v, 0))
def snitch(start_time=None):
if start_time is None:
snitch.delay(time())
return
snitch_id = settings.SNITCH_ID
totalms = int((time() - start_time) * 1000)
statsd.timing('news.tasks.snitch.timing', totalms)
requests.post('https://nosnch.in/{}'.format(snitch_id), data={
'm': totalms,
})
def estimates_report():
start = time.time()
d = item_counts()
# item counts
statsd.gauge('items.open.sm', d['open_sm_count'])
# hour estimates
statsd.gauge('estimates.sm', d['estimates_sm'])
statsd.gauge('estimates.non_sm', d['estimates_non_sm'])
end = time.time()
statsd.timing('celery.estimates_report', int((end - start) * 1000))
def on_task_postrun(sender=None, task_id=None, task=None, **kwds):
"""
Handle Celery ``task_postrun`` signals.
"""
# Increase statsd counter.
statsd.incr('celery.%s.done' % task.name)
# Log duration.
start_time = _task_start_times.pop(task_id, False)
if start_time:
ms = int((time.time() - start_time) * 1000)
statsd.timing('celery.%s.runtime' % task.name, ms)
def raw(self):
qs = self._build_query()
es = get_es()
try:
with statsd.timer('search.es.timer') as timer:
hits = es.search(qs, self.index, self.type._meta.db_table)
except Exception:
log.error(qs)
raise
statsd.timing('search.es.took', hits['took'])
log.debug('[%s] [%s] %s' % (hits['took'], timer.ms, qs))
return hits
def estimates_report():
start = time.time()
d = item_counts()
# item counts
statsd.gauge("items.open.sm", d["open_sm_count"])
# hour estimates
statsd.gauge("estimates.sm", d["estimates_sm"])
statsd.gauge("estimates.non_sm", d["estimates_non_sm"])
end = time.time()
statsd.timing("celery.estimates_report", int((end - start) * 1000))
def from_upload(cls, upload, addon, platforms, send_signal=True,
source=None, is_beta=False):
data = utils.parse_addon(upload, addon)
try:
license = addon.versions.latest().license_id
except Version.DoesNotExist:
license = None
max_len = cls._meta.get_field_by_name('_developer_name')[0].max_length
developer = data.get('developer_name', '')[:max_len]
v = cls.objects.create(
addon=addon,
version=data['version'],
license_id=license,
_developer_name=developer,
source=source
)
log.info('New version: %r (%s) from %r' % (v, v.id, upload))
AV = ApplicationsVersions
for app in data.get('apps', []):
AV(version=v, min=app.min, max=app.max,
application=app.id).save()
if addon.type == amo.ADDON_SEARCH:
# Search extensions are always for all platforms.
platforms = [amo.PLATFORM_ALL.id]
else:
platforms = cls._make_safe_platform_files(platforms)
for platform in platforms:
File.from_upload(upload, v, platform, parse_data=data,
is_beta=is_beta)
v.disable_old_files()
# After the upload has been copied to all platforms, remove the upload.
storage.delete(upload.path)
if send_signal:
version_uploaded.send(sender=v)
# Track the time it took from first upload through validation
# (and whatever else) until a version was created.
upload_start = utc_millesecs_from_epoch(upload.created)
now = datetime.datetime.now()
now_ts = utc_millesecs_from_epoch(now)
upload_time = now_ts - upload_start
log.info('Time for version {version} creation from upload: {delta}; '
'created={created}; now={now}'
.format(delta=upload_time, version=v,
created=upload.created, now=now))
statsd.timing('devhub.version_created_from_upload', upload_time)
return v
def on_postrun(self, sender, **kw):
# sender is the task object. task_id in here.
pending = self.redis.hincrby(self.pending, sender.name, -1)
# Clamp pending at 0. Tasks could be coming in before we started
# tracking.
if pending < 0:
self.redis.hset(self.pending, sender.name, 0)
self.redis.hincrby(self.run, sender.name, 1)
start = self.redis.hget(self.timer, kw['task_id'])
if start:
t = (time.time() - float(start)) * 1000
statsd.timing('tasks.%s' % sender.name, int(t))
def _process_summaries(start, keys):
calculated = {
'network': keys['window.performance.timing.responseStart'] - start,
'app': keys['window.performance.timing.domLoading'] -
keys['window.performance.timing.responseStart'],
'dom': keys['window.performance.timing.domComplete'] -
keys['window.performance.timing.domLoading'],
'rendering': keys['window.performance.timing.loadEventEnd'] -
keys['window.performance.timing.domComplete'],
}
for k, v in calculated.items():
# If loadEventEnd still does not get populated, we could end up with
# negative numbers here.
statsd.timing('window.performance.calculated.%s' % k, max(v, 0))
def track_task_run_time(task_id, task, **kw):
timer = TaskTimer()
start_time = cache.get(timer.cache_key(task_id))
if start_time is None:
log.info('could not track task run time; id={id}; name={name}; '
'current_dt={current_dt}'
.format(id=task_id, name=task.name,
current_dt=timer.current_datetime))
else:
run_time = timer.current_epoch_ms - start_time
log.info('tracking task run time; id={id}; name={name}; '
'run_time={run_time}; current_dt={current_dt}'
.format(id=task_id, name=task.name,
current_dt=timer.current_datetime,
run_time=run_time))
statsd.timing('tasks.{}'.format(task.name), run_time)
cache.delete(timer.cache_key(task_id))
def process_triggers(use_statsd=False, function_logger=None):
"""
Process all triggers that are ready for processing.
:param bool use_statsd: whether to use_statsd
:return: None
"""
process_async = getattr(settings, 'DJTRIGGERS_ASYNC_HANDLING', False)
# Get all triggers that need to be processed
for model in apps.get_models():
# Check whether it's a trigger
if not issubclass(model, Trigger) or getattr(model, 'typed', None) is None or isabstract(model):
continue
# Get all triggers of this type that need to be processed
triggers = model.objects.filter(Q(process_after__isnull=True) | Q(process_after__lt=timezone.now()),
date_processed__isnull=True)
# Process each trigger
for trigger in triggers:
try:
# Process the trigger, either synchronously or in a Celery task
if process_async:
process_trigger.apply_async((trigger.id, trigger._meta.app_label, trigger.__class__.__name__),
{'use_statsd': use_statsd},
max_retries=getattr(settings, 'DJTRIGGERS_CELERY_TASK_MAX_RETRIES', 0))
else:
trigger.process()
# Send stats to statsd if necessary
if use_statsd:
from django_statsd.clients import statsd
statsd.incr('triggers.{}.processed'.format(trigger.trigger_type))
if trigger.date_processed and trigger.process_after:
statsd.timing('triggers.{}.process_delay_seconds'.format(trigger.trigger_type),
(trigger.date_processed - trigger.process_after).total_seconds())
# The trigger didn't need processing yet
except ProcessLaterError:
pass
# The trigger raised an (expected) error while processing
except ProcessError:
pass
# In case a trigger got removed (manually or some process), deal with it
except Trigger.DoesNotExist as e:
logger.info(e)
def get_localepicker(self):
"""
For a file that is part of a language pack, extract
the chrome/localepicker.properties file and return as
a string.
"""
start = time.time()
zip = SafeZip(self.file_path, validate=False)
try:
is_valid = zip.is_valid()
except (zipfile.BadZipfile, IOError):
is_valid = False
if not is_valid:
return ''
try:
manifest = zip.read('chrome.manifest')
except KeyError as e:
log.info('No file named: chrome.manifest in file: %s' % self.pk)
return ''
res = self._get_localepicker.search(manifest)
if not res:
log.error('Locale browser not in chrome.manifest: %s' % self.pk)
return ''
try:
p = res.groups()[1]
if 'localepicker.properties' not in p:
p = os.path.join(p, 'localepicker.properties')
res = zip.extract_from_manifest(p)
except (zipfile.BadZipfile, IOError) as e:
log.error('Error unzipping: %s, %s in file: %s' % (p, e, self.pk))
return ''
except (ValueError, KeyError) as e:
log.error('No file named: %s in file: %s' % (e, self.pk))
return ''
end = time.time() - start
log.info('Extracted localepicker file: %s in %.2fs' %
(self.pk, end))
statsd.timing('files.extract.localepicker', (end * 1000))
return res
def wait_to_start(request):
"""
Wait until the transaction is in a ready state.
The transaction was started previously during the buy flow in the
background from webpay.pay.tasks.
Serve JS that polls for transaction state.
When ready, redirect to the Bango payment URL using
the generated billing configuration ID.
"""
trans_id = request.session.get('trans_id', None)
if not trans_id:
# This seems like a seriously problem but maybe there is just a race
# condition. If we see a lot of these in the logs it means the
# payment will never complete so we should keep an eye on it.
log.error('wait_to_start() session trans_id {t} was None'
.format(t=trans_id))
try:
statsd.incr('purchase.payment_time.retry')
with statsd.timer('purchase.payment_time.get_transation'):
trans = solitude.get_transaction(trans_id)
except ObjectDoesNotExist:
trans = {'status': None}
if trans['status'] in constants.STATUS_ENDED:
statsd.incr('purchase.payment_time.failure')
log.exception('Attempt to restart finished transaction {0} '
'with status {1}'.format(trans_id, trans['status']))
return system_error(request, code=msg.TRANS_ENDED)
if trans['status'] == constants.STATUS_PENDING:
statsd.incr('purchase.payment_time.success')
payment_start = request.session.get('payment_start', False)
if payment_start:
delta = int((time.time() - float(payment_start)) * 1000)
statsd.timing('purchase.payment_time.duration', delta)
# Dump any messages so we don't show them later.
clear_messages(request)
# The transaction is ready; no need to wait for it.
url = get_payment_url(trans)
log.info('immediately redirecting to payment URL {url} '
'for trans {tr}'.format(url=url, tr=trans))
return http.HttpResponseRedirect(url)
return render(request, 'pay/wait-to-start.html')
def raw(self):
build_body = self._build_query()
es = get_es()
try:
with statsd.timer('search.es.timer') as timer:
hits = es.search(
body=build_body,
index=self.index,
doc_type=self.type._meta.db_table,
)
except Exception:
log.error(build_body)
raise
statsd.timing('search.es.took', hits['took'])
log.debug('[%s] [%s] %s' % (hits['took'], timer.ms, build_body))
return hits
def raw(self):
build_body = self._build_query()
es = get_es()
try:
with statsd.timer('search.es.timer') as timer:
hits = es.search(
body=build_body,
index=self.index,
doc_type=self.type._meta.db_table,
)
except Exception:
log.error(build_body)
raise
statsd.timing('search.es.took', hits['took'])
log.info('[%s] [%s] %s' % (hits['took'], timer.ms, build_body))
return hits
def _process_summaries(start, keys):
calculated = {
'network':
keys['window.performance.timing.responseStart'] - start,
'app':
keys['window.performance.timing.domLoading'] -
keys['window.performance.timing.responseStart'],
'dom':
keys['window.performance.timing.domComplete'] -
keys['window.performance.timing.domLoading'],
'rendering':
keys['window.performance.timing.loadEventEnd'] -
keys['window.performance.timing.domComplete'],
}
for k, v in calculated.items():
# If loadEventEnd still does not get populated, we could end up with
# negative numbers here.
statsd.timing('window.performance.calculated.%s' % k, max(v, 0))
def get_localepicker(self):
"""
For a file that is part of a language pack, extract
the chrome/localepicker.properties file and return as
a string.
"""
start = time.time()
zip = SafeZip(self.file_path, validate=False)
try:
is_valid = zip.is_valid()
except (zipfile.BadZipfile, IOError):
is_valid = False
if not is_valid:
return ''
try:
manifest = zip.read('chrome.manifest')
except KeyError as e:
log.info('No file named: chrome.manifest in file: %s' % self.pk)
return ''
res = self._get_localepicker.search(manifest)
if not res:
log.error('Locale browser not in chrome.manifest: %s' % self.pk)
return ''
try:
p = res.groups()[1]
if 'localepicker.properties' not in p:
p = os.path.join(p, 'localepicker.properties')
res = zip.extract_from_manifest(p)
except (zipfile.BadZipfile, IOError) as e:
log.error('Error unzipping: %s, %s in file: %s' % (p, e, self.pk))
return ''
except (ValueError, KeyError) as e:
log.error('No file named: %s in file: %s' % (e, self.pk))
return ''
end = time.time() - start
log.info('Extracted localepicker file: %s in %.2fs' % (self.pk, end))
statsd.timing('files.extract.localepicker', (end * 1000))
return res
def wrapped(self, *args, **kwargs):
start_time = kwargs.pop('start_time', None)
if start_time and not self.request.retries:
total_time = int((time() - start_time) * 1000)
statsd.timing(self.name + '.timing', total_time)
statsd.incr(self.name + '.total')
statsd.incr('news.tasks.all_total')
if settings.MAINTENANCE_MODE and self.name not in MAINTENANCE_EXEMPT:
if not settings.READ_ONLY_MODE:
# record task for later
QueuedTask.objects.create(
name=self.name,
args=args,
kwargs=kwargs,
)
statsd.incr(self.name + '.queued')
else:
statsd.incr(self.name + '.not_queued')
return
try:
return func(*args, **kwargs)
except (IOError, NewsletterException, requests.RequestException,
sfapi.SalesforceError, RetryTask) as e:
# These could all be connection issues, so try again later.
# IOError covers URLError and SSLError.
if ignore_error(e):
return
try:
if not (isinstance(e, RetryTask)
or ignore_error_post_retry(e)):
sentry_client.captureException(tags={'action': 'retried'})
raise self.retry(countdown=2**(self.request.retries + 1) * 60)
except self.MaxRetriesExceededError:
statsd.incr(self.name + '.retry_max')
statsd.incr('news.tasks.retry_max_total')
# don't bubble certain errors
if ignore_error_post_retry(e):
return
sentry_client.captureException()
def wrapped(*args, **kwargs):
starttime = time()
e = None
try:
resp = f(*args, **kwargs)
except NewsletterException as e:
pass
except Exception:
raise
totaltime = int((time() - starttime) * 1000)
statsd.timing(prefix + '.timing', totaltime)
statsd.timing(prefix + '.{}.timing'.format(f.__name__), totaltime)
statsd.incr(prefix + '.count')
statsd.incr(prefix + '.{}.count'.format(f.__name__))
if e:
raise
else:
return resp
def track_task_run_time(task_id, task, **kw):
timer = TaskTimer()
start_time = cache.get(timer.cache_key(task_id))
if start_time is None:
log.info('could not track task run time; id={id}; name={name}; '
'current_dt={current_dt}'.format(
id=task_id,
name=task.name,
current_dt=timer.current_datetime))
else:
run_time = timer.current_epoch_ms - start_time
log.info('tracking task run time; id={id}; name={name}; '
'run_time={run_time}; current_dt={current_dt}'.format(
id=task_id,
name=task.name,
current_dt=timer.current_datetime,
run_time=run_time))
statsd.timing('tasks.{}'.format(task.name), run_time)
cache.delete(timer.cache_key(task_id))
def trans_start_url(request):
"""
JSON handler to get the Bango payment URL to start a transaction.
"""
try:
statsd.incr('purchase.payment_time.retry')
with statsd.timer('purchase.payment_time.get_transation'):
trans = solitude.get_transaction(request.session['trans_id'])
except ObjectDoesNotExist:
trans = {'status': None}
data = {'url': None, 'status': trans['status']}
if trans['status'] == constants.STATUS_PENDING:
statsd.incr('purchase.payment_time.success')
payment_start = request.session.get('payment_start', False)
if payment_start:
delta = int((time.time() - float(payment_start)) * 1000)
statsd.timing('purchase.payment_time.duration', delta)
data['url'] = provider.get_start_url(trans['uid_pay'])
return data
def new_execute_command(self, *args, **options):
# The key is the name of the command in lowercase
key = args[0].lower()
# Start the timer
start = time.time()
# Run the command
ret = self._old_execute_command(*args, **options)
# Get the time
ms = int(round((time.time() - start) * 1000)) # delta in ms
# Log the stats
statsd.timing('redis.execute', ms, SAMPLE_RATE)
statsd.timing('redis.execute.' + key, ms, SAMPLE_RATE)
# Done
return ret
def wrapped(*args, **kwargs):
starttime = time()
e = None
try:
resp = f(*args, **kwargs)
except NewsletterException as e:
pass
except Exception:
raise
totaltime = int((time() - starttime) * 1000)
statsd.timing(prefix + '.timing', totaltime)
statsd.timing(prefix + '.{}.timing'.format(f.__name__), totaltime)
statsd.incr(prefix + '.count')
statsd.incr(prefix + '.{}.count'.format(f.__name__))
if e:
raise
else:
return resp
def wait_to_start(request):
"""
Wait until the transaction is in a ready state.
The transaction was started previously during the buy flow in the
background from webpay.pay.tasks.
Serve JS that polls for transaction state.
When ready, redirect to the Bango payment URL using
the generated billing configuration ID.
"""
trans_id = request.session.get('trans_id', None)
if not trans_id:
# This seems like a seriously problem but maybe there is just a race
# condition. If we see a lot of these in the logs it means the
# payment will never complete so we should keep an eye on it.
log.error(
'wait_to_start() session trans_id {t} was None'.format(t=trans_id))
try:
statsd.incr('purchase.payment_time.retry')
with statsd.timer('purchase.payment_time.get_transation'):
trans = solitude.get_transaction(trans_id)
except ObjectDoesNotExist:
trans = {'status': None}
if trans['status'] in constants.STATUS_ENDED:
statsd.incr('purchase.payment_time.failure')
log.exception('Attempt to restart finished transaction {0} '
'with status {1}'.format(trans_id, trans['status']))
return system_error(request, code=msg.TRANS_ENDED)
if trans['status'] == constants.STATUS_PENDING:
statsd.incr('purchase.payment_time.success')
payment_start = request.session.get('payment_start', False)
if payment_start:
delta = int((time.time() - float(payment_start)) * 1000)
statsd.timing('purchase.payment_time.duration', delta)
# Dump any messages so we don't show them later.
clear_messages(request)
# The transaction is ready; no need to wait for it.
return http.HttpResponseRedirect(get_payment_url(trans))
return render(request, 'pay/wait-to-start.html')
def trans_start_url(request):
"""
JSON handler to get the provider payment URL to start a transaction.
"""
trans = None
trans_id = request.session.get('trans_id')
data = {'url': None, 'status': None, 'provider': None}
if not trans_id:
log.error('trans_start_url(): no transaction ID in session')
return http.HttpResponseBadRequest()
try:
statsd.incr('purchase.payment_time.retry')
with statsd.timer('purchase.payment_time.get_transaction'):
trans = client.get_transaction(trans_id)
data['status'] = trans['status']
data['provider'] = constants.PROVIDERS_INVERTED.get(trans['provider'])
except ObjectDoesNotExist:
log.error('trans_start_url() transaction does not exist: {t}'
.format(t=trans_id))
if data['status'] == constants.STATUS_PENDING:
statsd.incr('purchase.payment_time.success')
payment_start = request.session.get('payment_start', False)
if payment_start:
delta = int((time.time() - float(payment_start)) * 1000)
statsd.timing('purchase.payment_time.duration', delta)
url = get_payment_url(trans)
log.info('async call got payment URL {url} for trans {tr}'
.format(url=url, tr=trans))
data['url'] = url
if trans and trans['status'] == constants.STATUS_ERRORED:
statsd.incr('purchase.payment_time.errored')
log.exception('Purchase configuration failed: {0} with status {1}'
.format(trans_id, trans['status']))
return system_error(
request,
code=getattr(msg, trans.get('status_reason', 'UNEXPECTED_ERROR'))
)
return data
def _handle_execution_success(self, use_statsd=False):
"""
Handle execution success of the trigger
:param bool use_statsd: whether to use statsd
:return: None
"""
if self.date_processed is None:
now = timezone.now()
self.date_processed = now
# Send stats to statsd if necessary
if use_statsd:
from django_statsd.clients import statsd
statsd.incr('triggers.{trigger_type}.processed'.format(trigger_type=self.trigger_type))
if self.date_processed and self.process_after:
statsd.timing('triggers.{trigger_type}.process_delay_seconds'.format(trigger_type=self.trigger_type),
(self.date_processed - self.process_after).total_seconds())
self.successful = True
self.save()
def trans_start_url(request):
"""
JSON handler to get the Bango payment URL to start a transaction.
"""
try:
statsd.incr('purchase.payment_time.retry')
with statsd.timer('purchase.payment_time.get_transaction'):
trans = solitude.get_transaction(request.session['trans_id'])
except ObjectDoesNotExist:
log.error('trans_start_url() transaction does not exist: {t}'.format(
t=request.session['trans_id']))
trans = {'status': None}
data = {'url': None, 'status': trans['status']}
if trans['status'] == constants.STATUS_PENDING:
statsd.incr('purchase.payment_time.success')
payment_start = request.session.get('payment_start', False)
if payment_start:
delta = int((time.time() - float(payment_start)) * 1000)
statsd.timing('purchase.payment_time.duration', delta)
data['url'] = get_payment_url(trans)
return data
def handle_upload_validation_result(results, upload_pk, channel, is_mozilla_signed):
"""Annotate a set of validation results and save them to the given
FileUpload instance.
"""
upload = FileUpload.objects.get(pk=upload_pk)
upload.validation = json.dumps(results)
upload.save() # We want to hit the custom save().
# Track the time it took from first upload through validation
# until the results were processed and saved.
upload_start = utc_millesecs_from_epoch(upload.created)
now = datetime.datetime.now()
now_ts = utc_millesecs_from_epoch(now)
delta = now_ts - upload_start
statsd.timing('devhub.validation_results_processed', delta)
if not storage.exists(upload.path):
# TODO: actually fix this so we can get stats. It seems that
# the file maybe gets moved but it needs more investigation.
log.warning(
'Scaled upload stats were not tracked. File is '
'missing: {}'.format(upload.path)
)
return
size = Decimal(storage.size(upload.path))
megabyte = Decimal(1024 * 1024)
# Stash separate metrics for small / large files.
quantifier = 'over' if size > megabyte else 'under'
statsd.timing(
'devhub.validation_results_processed_{}_1mb'.format(quantifier), delta
)
# Scale the upload / processing time by package size (in MB)
# so we can normalize large XPIs which naturally take longer to validate.
scaled_delta = None
size_in_mb = size / megabyte
if size > 0:
# If the package is smaller than 1MB, don't scale it. This should
# help account for validator setup time.
unit = size_in_mb if size > megabyte else Decimal(1)
scaled_delta = Decimal(delta) / unit
statsd.timing('devhub.validation_results_processed_per_mb', scaled_delta)
log.info(
'Time to process and save upload validation; '
'upload.pk={upload}; processing_time={delta}; '
'scaled_per_mb={scaled}; upload_size_in_mb={size_in_mb}; '
'created={created}; now={now}'.format(
delta=delta,
upload=upload.pk,
created=upload.created,
now=now,
scaled=scaled_delta,
size_in_mb=size_in_mb,
)
)
def _record_time(self, request):
if hasattr(request, '_start_time'):
ms = int((time.time() - request._start_time) * 1000)
data = dict(module=request._view_module, name=request._view_name,
method=request.method)
statsd.timing('view.{module}.{name}.{method}'.format(**data), ms)
if getattr(settings, 'STATSD_VIEW_TIMER_DETAILS', True):
statsd.timing('view.{module}.{method}'.format(**data), ms)
statsd.timing('view.{method}'.format(**data), ms)
def _record_time(self, request):
if hasattr(request, '_start_time'):
ms = int((time.time() - request._start_time) * 1000)
data = dict(module=request._view_module,
name=request._view_name,
method=request.method)
statsd.timing('view.{module}.{name}.{method}'.format(**data), ms)
statsd.timing('view.{module}.{method}'.format(**data), ms)
statsd.timing('view.{method}'.format(**data), ms)
def _record_time(self, request):
pre = 'api' if getattr(request, 'API', False) else 'view'
if hasattr(request, '_start_time'):
ms = int((time.time() - request._start_time) * 1000)
data = {'method': request.method,
'module': request._view_module,
'name': request._view_name,
'pre': pre}
statsd.timing('{pre}.{module}.{name}.{method}'.format(**data), ms)
statsd.timing('{pre}.{module}.{method}'.format(**data), ms)
statsd.timing('{pre}.{method}'.format(**data), ms)
def handle_upload_validation_result(results, upload_pk, channel,
is_mozilla_signed):
"""Annotate a set of validation results and save them to the given
FileUpload instance."""
upload = FileUpload.objects.get(pk=upload_pk)
# Restrictions applying to new legacy submissions apply if:
# - It's the very first upload (there is no addon id yet)
# - It's the first upload in that channel
is_new_upload = (
not upload.addon_id
or not upload.addon.find_latest_version(channel=channel, exclude=()))
# Annotate results with potential legacy add-ons restrictions.
if not is_mozilla_signed:
results = annotate_legacy_addon_restrictions(
results=results, is_new_upload=is_new_upload)
annotate_legacy_langpack_restriction(results=results)
# Check for API keys in submissions.
# Make sure it is extension-like, e.g. no LWT or search plugin
try:
results = check_for_api_keys_in_file(results=results, upload=upload)
except (ValidationError, BadZipfile, IOError):
pass
# Annotate results with potential webext warnings on new versions.
if upload.addon_id and upload.version:
results = annotate_webext_incompatibilities(
results=results,
file_=None,
addon=upload.addon,
version_string=upload.version,
channel=channel)
upload.validation = json.dumps(results)
upload.save() # We want to hit the custom save().
# Track the time it took from first upload through validation
# until the results were processed and saved.
upload_start = utc_millesecs_from_epoch(upload.created)
now = datetime.datetime.now()
now_ts = utc_millesecs_from_epoch(now)
delta = now_ts - upload_start
statsd.timing('devhub.validation_results_processed', delta)
if not storage.exists(upload.path):
# TODO: actually fix this so we can get stats. It seems that
# the file maybe gets moved but it needs more investigation.
log.warning('Scaled upload stats were not tracked. File is '
'missing: {}'.format(upload.path))
return
size = Decimal(storage.size(upload.path))
megabyte = Decimal(1024 * 1024)
# Stash separate metrics for small / large files.
quantifier = 'over' if size > megabyte else 'under'
statsd.timing(
'devhub.validation_results_processed_{}_1mb'.format(quantifier), delta)
# Scale the upload / processing time by package size (in MB)
# so we can normalize large XPIs which naturally take longer to validate.
scaled_delta = None
size_in_mb = size / megabyte
if size > 0:
# If the package is smaller than 1MB, don't scale it. This should
# help account for validator setup time.
unit = size_in_mb if size > megabyte else Decimal(1)
scaled_delta = Decimal(delta) / unit
statsd.timing('devhub.validation_results_processed_per_mb',
scaled_delta)
log.info('Time to process and save upload validation; '
'upload.pk={upload}; processing_time={delta}; '
'scaled_per_mb={scaled}; upload_size_in_mb={size_in_mb}; '
'created={created}; now={now}'.format(delta=delta,
upload=upload.pk,
created=upload.created,
now=now,
scaled=scaled_delta,
size_in_mb=size_in_mb))
def from_upload(cls,
upload,
addon,
platforms,
channel,
send_signal=True,
source=None,
is_beta=False,
parsed_data=None):
from olympia.addons.models import AddonFeatureCompatibility
if addon.status == amo.STATUS_DISABLED:
raise VersionCreateError(
'Addon is Mozilla Disabled; no new versions are allowed.')
if parsed_data is None:
parsed_data = utils.parse_addon(upload, addon)
license_id = None
if channel == amo.RELEASE_CHANNEL_LISTED:
previous_version = addon.find_latest_version(channel=channel,
exclude=())
if previous_version and previous_version.license_id:
license_id = previous_version.license_id
version = cls.objects.create(
addon=addon,
version=parsed_data['version'],
license_id=license_id,
source=source,
channel=channel,
)
log.info('New version: %r (%s) from %r' %
(version, version.id, upload))
activity.log_create(amo.LOG.ADD_VERSION, version, addon)
# Update the add-on e10s compatibility since we're creating a new
# version that may change that.
e10s_compatibility = parsed_data.get('e10s_compatibility')
if e10s_compatibility is not None:
feature_compatibility = (
AddonFeatureCompatibility.objects.get_or_create(
addon=addon)[0])
feature_compatibility.update(e10s=e10s_compatibility)
compatible_apps = {}
for app in parsed_data.get('apps', []):
compatible_apps[app.appdata] = ApplicationsVersions(
version=version, min=app.min, max=app.max, application=app.id)
compatible_apps[app.appdata].save()
# See #2828: sometimes when we generate the filename(s) below, in
# File.from_upload(), cache-machine is confused and has trouble
# fetching the ApplicationsVersions that were just created. To work
# around this we pre-generate version.compatible_apps and avoid the
# queries completely.
version.compatible_apps = compatible_apps
if addon.type == amo.ADDON_SEARCH:
# Search extensions are always for all platforms.
platforms = [amo.PLATFORM_ALL.id]
else:
platforms = cls._make_safe_platform_files(platforms)
for platform in platforms:
File.from_upload(upload,
version,
platform,
parsed_data=parsed_data,
is_beta=is_beta)
version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW])
version.disable_old_files()
# After the upload has been copied to all platforms, remove the upload.
storage.delete(upload.path)
if send_signal:
version_uploaded.send(sender=version)
# Track the time it took from first upload through validation
# (and whatever else) until a version was created.
upload_start = utc_millesecs_from_epoch(upload.created)
now = datetime.datetime.now()
now_ts = utc_millesecs_from_epoch(now)
upload_time = now_ts - upload_start
log.info('Time for version {version} creation from upload: {delta}; '
'created={created}; now={now}'.format(delta=upload_time,
version=version,
created=upload.created,
now=now))
statsd.timing('devhub.version_created_from_upload', upload_time)
return version
def from_upload(cls,
upload,
addon,
selected_apps,
channel,
parsed_data=None):
"""
Create a Version instance and corresponding File(s) from a
FileUpload, an Addon, a list of compatible app ids, a channel id and
the parsed_data generated by parse_addon().
Note that it's the caller's responsability to ensure the file is valid.
We can't check for that here because an admin may have overridden the
validation results.
"""
from olympia.addons.models import AddonReviewerFlags
from olympia.addons.utils import RestrictionChecker
from olympia.git.utils import create_git_extraction_entry
assert parsed_data is not None
if addon.status == amo.STATUS_DISABLED:
raise VersionCreateError(
'Addon is Mozilla Disabled; no new versions are allowed.')
if upload.addon and upload.addon != addon:
raise VersionCreateError(
'FileUpload was made for a different Addon')
if not upload.user or not upload.ip_address or not upload.source:
raise VersionCreateError(
'FileUpload does not have some required fields')
if not upload.user.last_login_ip or not upload.user.email:
raise VersionCreateError(
'FileUpload user does not have some required fields')
license_id = None
if channel == amo.RELEASE_CHANNEL_LISTED:
previous_version = addon.find_latest_version(channel=channel,
exclude=())
if previous_version and previous_version.license_id:
license_id = previous_version.license_id
approval_notes = None
if parsed_data.get('is_mozilla_signed_extension'):
approval_notes = (
'This version has been signed with Mozilla internal certificate.'
)
version = cls.objects.create(
addon=addon,
approval_notes=approval_notes,
version=parsed_data['version'],
license_id=license_id,
channel=channel,
)
email = upload.user.email if upload.user and upload.user.email else ''
with core.override_remote_addr(upload.ip_address):
# The following log statement is used by foxsec-pipeline.
# We override the IP because it might be called from a task and we
# want the original IP from the submitter.
log.info(
f'New version: {version!r} ({version.id}) from {upload!r}',
extra={
'email': email,
'guid': addon.guid,
'upload': upload.uuid.hex,
'user_id': upload.user_id,
'from_api': upload.source == amo.UPLOAD_SOURCE_API,
},
)
activity.log_create(amo.LOG.ADD_VERSION,
version,
addon,
user=upload.user or get_task_user())
if addon.type == amo.ADDON_STATICTHEME:
# We don't let developers select apps for static themes
selected_apps = [app.id for app in amo.APP_USAGE]
compatible_apps = {}
for app in parsed_data.get('apps', []):
if app.id not in selected_apps:
# If the user chose to explicitly deselect Firefox for Android
# we're not creating the respective `ApplicationsVersions`
# which will have this add-on then be listed only for
# Firefox specifically.
continue
compatible_apps[app.appdata] = ApplicationsVersions(
version=version, min=app.min, max=app.max, application=app.id)
compatible_apps[app.appdata].save()
# Pre-generate _compatible_apps property to avoid accidentally
# triggering queries with that instance later.
version._compatible_apps = compatible_apps
# Create relevant file and update the all_files cached property on the
# Version, because we might need it afterwards.
version.all_files = [
File.from_upload(
upload=upload,
version=version,
parsed_data=parsed_data,
)
]
version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW])
version.disable_old_files()
# After the upload has been copied to its permanent location, delete it
# from storage. Keep the FileUpload instance (it gets cleaned up by a
# cron eventually some time after its creation, in amo.cron.gc()),
# making sure it's associated with the add-on instance.
storage.delete(upload.path)
upload.path = ''
if upload.addon is None:
upload.addon = addon
upload.save()
version_uploaded.send(instance=version, sender=Version)
if version.is_webextension:
if (waffle.switch_is_active('enable-yara')
or waffle.switch_is_active('enable-customs')
or waffle.switch_is_active('enable-wat')):
ScannerResult.objects.filter(upload_id=upload.id).update(
version=version)
if waffle.switch_is_active('enable-uploads-commit-to-git-storage'):
# Schedule this version for git extraction.
transaction.on_commit(
lambda: create_git_extraction_entry(version=version))
# Generate a preview and icon for listed static themes
if (addon.type == amo.ADDON_STATICTHEME
and channel == amo.RELEASE_CHANNEL_LISTED):
theme_data = parsed_data.get('theme', {})
generate_static_theme_preview(theme_data, version.pk)
# Reset add-on reviewer flags to disable auto-approval and require
# admin code review if the package has already been signed by mozilla.
reviewer_flags_defaults = {}
is_mozilla_signed = parsed_data.get('is_mozilla_signed_extension')
if upload.validation_timeout:
reviewer_flags_defaults['needs_admin_code_review'] = True
if is_mozilla_signed and addon.type != amo.ADDON_LPAPP:
reviewer_flags_defaults['needs_admin_code_review'] = True
reviewer_flags_defaults['auto_approval_disabled'] = True
# Check if the approval should be restricted
if not RestrictionChecker(upload=upload).is_auto_approval_allowed():
flag = ('auto_approval_disabled'
if channel == amo.RELEASE_CHANNEL_LISTED else
'auto_approval_disabled_unlisted')
reviewer_flags_defaults[flag] = True
if reviewer_flags_defaults:
AddonReviewerFlags.objects.update_or_create(
addon=addon, defaults=reviewer_flags_defaults)
# Authors need to be notified about auto-approval delay again since
# they are submitting a new version.
addon.reset_notified_about_auto_approval_delay()
# Track the time it took from first upload through validation
# (and whatever else) until a version was created.
upload_start = utc_millesecs_from_epoch(upload.created)
now = datetime.datetime.now()
now_ts = utc_millesecs_from_epoch(now)
upload_time = now_ts - upload_start
log.info('Time for version {version} creation from upload: {delta}; '
'created={created}; now={now}'.format(delta=upload_time,
version=version,
created=upload.created,
now=now))
statsd.timing('devhub.version_created_from_upload', upload_time)
return version
def from_upload(cls, upload, addon, selected_apps, channel,
parsed_data=None):
"""
Create a Version instance and corresponding File(s) from a
FileUpload, an Addon, a list of compatible app ids, a channel id and
the parsed_data generated by parse_addon().
Note that it's the caller's responsability to ensure the file is valid.
We can't check for that here because an admin may have overridden the
validation results.
"""
from olympia.git.utils import create_git_extraction_entry
assert parsed_data is not None
if addon.status == amo.STATUS_DISABLED:
raise VersionCreateError(
'Addon is Mozilla Disabled; no new versions are allowed.')
license_id = None
if channel == amo.RELEASE_CHANNEL_LISTED:
previous_version = addon.find_latest_version(
channel=channel, exclude=())
if previous_version and previous_version.license_id:
license_id = previous_version.license_id
approval_notes = None
if parsed_data.get('is_mozilla_signed_extension'):
approval_notes = (u'This version has been signed with '
u'Mozilla internal certificate.')
version = cls.objects.create(
addon=addon,
approval_notes=approval_notes,
version=parsed_data['version'],
license_id=license_id,
channel=channel,
)
email = upload.user.email if upload.user and upload.user.email else ''
with core.override_remote_addr(upload.ip_address):
log.info(
'New version: %r (%s) from %r' % (version, version.id, upload),
extra={
'email': email,
'guid': addon.guid,
'upload': upload.uuid.hex,
'user_id': upload.user_id,
'from_api': upload.source == amo.UPLOAD_SOURCE_API,
}
)
activity.log_create(
amo.LOG.ADD_VERSION, version, addon,
user=upload.user or get_task_user())
if addon.type == amo.ADDON_STATICTHEME:
# We don't let developers select apps for static themes
selected_apps = [app.id for app in amo.APP_USAGE]
compatible_apps = {}
for app in parsed_data.get('apps', []):
if app.id not in selected_apps:
# If the user chose to explicitly deselect Firefox for Android
# we're not creating the respective `ApplicationsVersions`
# which will have this add-on then be listed only for
# Firefox specifically.
continue
compatible_apps[app.appdata] = ApplicationsVersions(
version=version, min=app.min, max=app.max, application=app.id)
compatible_apps[app.appdata].save()
# See #2828: sometimes when we generate the filename(s) below, in
# File.from_upload(), cache-machine is confused and has trouble
# fetching the ApplicationsVersions that were just created. To work
# around this we pre-generate version.compatible_apps and avoid the
# queries completely.
version._compatible_apps = compatible_apps
# For backwards compatibility. We removed specific platform
# support during submission but we don't handle it any different
# beyond that yet. That means, we're going to simply set it
# to `PLATFORM_ALL` and also have the backend create separate
# files for each platform. Cleaning that up is another step.
# Given the timing on this, we don't care about updates to legacy
# add-ons as well.
# Create relevant file and update the all_files cached property on the
# Version, because we might need it afterwards.
version.all_files = [File.from_upload(
upload=upload, version=version, platform=amo.PLATFORM_ALL.id,
parsed_data=parsed_data
)]
version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW])
version.disable_old_files()
# After the upload has been copied to all platforms, remove the upload.
storage.delete(upload.path)
upload.path = ''
upload.save()
version_uploaded.send(instance=version, sender=Version)
if version.is_webextension:
if (
waffle.switch_is_active('enable-yara') or
waffle.switch_is_active('enable-customs') or
waffle.switch_is_active('enable-wat')
):
ScannerResult.objects.filter(upload_id=upload.id).update(
version=version)
if waffle.switch_is_active('enable-uploads-commit-to-git-storage'):
# Schedule this version for git extraction.
transaction.on_commit(
lambda: create_git_extraction_entry(version=version)
)
# Generate a preview and icon for listed static themes
if (addon.type == amo.ADDON_STATICTHEME and
channel == amo.RELEASE_CHANNEL_LISTED):
theme_data = parsed_data.get('theme', {})
generate_static_theme_preview(theme_data, version.pk)
# Authors need to be notified about auto-approval delay again since
# they are submitting a new version.
addon.reset_notified_about_auto_approval_delay()
# Track the time it took from first upload through validation
# (and whatever else) until a version was created.
upload_start = utc_millesecs_from_epoch(upload.created)
now = datetime.datetime.now()
now_ts = utc_millesecs_from_epoch(now)
upload_time = now_ts - upload_start
log.info('Time for version {version} creation from upload: {delta}; '
'created={created}; now={now}'
.format(delta=upload_time, version=version,
created=upload.created, now=now))
statsd.timing('devhub.version_created_from_upload', upload_time)
return version
def handle_upload_validation_result(
results, upload_pk, channel, is_mozilla_signed):
"""Annotate a set of validation results and save them to the given
FileUpload instance."""
upload = FileUpload.objects.get(pk=upload_pk)
if waffle.switch_is_active('enable-yara') and results['errors'] == 0:
# Run Yara. This cannot be asynchronous because we have no way to know
# whether the task will complete before we attach a `Version` to it
# later in the submission process... Because we cannot use `chord`
# reliably right now (requires Celery 4.2+), this task is actually not
# run as a task, it's a simple function call.
#
# TODO: use `run_yara` as a task in the submission chord once it is
# possible. See: https://github.com/mozilla/addons-server/issues/12216
run_yara(upload.pk)
if waffle.switch_is_active('enable-customs') and results['errors'] == 0:
# Run customs. This cannot be asynchronous because we have no way to
# know whether the task will complete before we attach a `Version` to
# it later in the submission process... Because we cannot use `chord`
# reliably right now (requires Celery 4.2+), this task is actually not
# run as a task, it's a simple function call.
#
# TODO: use `run_customs` as a task in the submission chord once it is
# possible. See: https://github.com/mozilla/addons-server/issues/12217
run_customs(upload.pk)
if waffle.switch_is_active('enable-wat') and results['errors'] == 0:
# Run wat. This cannot be asynchronous because we have no way to know
# whether the task will complete before we attach a `Version` to it
# later in the submission process... Because we cannot use `chord`
# reliably right now (requires Celery 4.2+), this task is actually not
# run as a task, it's a simple function call.
#
# TODO: use `run_wat` as a task in the submission chord once it is
# possible. See: https://github.com/mozilla/addons-server/issues/12224
run_wat(upload.pk)
# Check for API keys in submissions.
# Make sure it is extension-like, e.g. no search plugin
try:
results = check_for_api_keys_in_file(results=results, upload=upload)
except (ValidationError, BadZipfile, IOError):
pass
# Annotate results with potential webext warnings on new versions.
if upload.addon_id and upload.version:
annotations.annotate_webext_incompatibilities(
results=results, file_=None, addon=upload.addon,
version_string=upload.version, channel=channel)
upload.validation = json.dumps(results)
upload.save() # We want to hit the custom save().
# Track the time it took from first upload through validation
# until the results were processed and saved.
upload_start = utc_millesecs_from_epoch(upload.created)
now = datetime.datetime.now()
now_ts = utc_millesecs_from_epoch(now)
delta = now_ts - upload_start
statsd.timing('devhub.validation_results_processed', delta)
if not storage.exists(upload.path):
# TODO: actually fix this so we can get stats. It seems that
# the file maybe gets moved but it needs more investigation.
log.warning('Scaled upload stats were not tracked. File is '
'missing: {}'.format(upload.path))
return
size = Decimal(storage.size(upload.path))
megabyte = Decimal(1024 * 1024)
# Stash separate metrics for small / large files.
quantifier = 'over' if size > megabyte else 'under'
statsd.timing(
'devhub.validation_results_processed_{}_1mb'.format(quantifier), delta)
# Scale the upload / processing time by package size (in MB)
# so we can normalize large XPIs which naturally take longer to validate.
scaled_delta = None
size_in_mb = size / megabyte
if size > 0:
# If the package is smaller than 1MB, don't scale it. This should
# help account for validator setup time.
unit = size_in_mb if size > megabyte else Decimal(1)
scaled_delta = Decimal(delta) / unit
statsd.timing('devhub.validation_results_processed_per_mb',
scaled_delta)
log.info('Time to process and save upload validation; '
'upload.pk={upload}; processing_time={delta}; '
'scaled_per_mb={scaled}; upload_size_in_mb={size_in_mb}; '
'created={created}; now={now}'
.format(delta=delta, upload=upload.pk,
created=upload.created, now=now,
scaled=scaled_delta, size_in_mb=size_in_mb))
def hours_logged_report():
start = time.time()
statsd.gauge("hours.one_week", hours_logged())
end = time.time()
statsd.timing('celery.hours_logged_report', int((end - start) * 1000))
def wrapped(self, *args, **kwargs):
start_time = kwargs.pop("start_time", None)
if start_time and not self.request.retries:
total_time = int((time() - start_time) * 1000)
statsd.timing(self.name + ".timing", total_time)
statsd.incr(self.name + ".total")
statsd.incr("news.tasks.all_total")
if settings.MAINTENANCE_MODE and self.name not in MAINTENANCE_EXEMPT:
if not settings.READ_ONLY_MODE:
# record task for later
QueuedTask.objects.create(
name=self.name,
args=args,
kwargs=kwargs,
)
statsd.incr(self.name + ".queued")
else:
statsd.incr(self.name + ".not_queued")
return
try:
return func(*args, **kwargs)
except (
IOError,
NewsletterException,
requests.RequestException,
RetryTask,
sfapi.SalesforceExpiredSession,
sfapi.SalesforceGeneralError,
sfapi.SalesforceRefusedRequest,
sfapi.SalesforceResourceNotFound,
sfapi.SalesforceAuthenticationFailed,
sfapi.SalesforceMalformedRequest,
SilverpopResponseException,
) as e:
# These could all be connection issues, so try again later.
# IOError covers URLError and SSLError.
if ignore_error(e):
with sentry_sdk.push_scope() as scope:
scope.set_tag("action", "ignored")
sentry_sdk.capture_exception()
return
try:
if not (isinstance(e, RetryTask)
or ignore_error_post_retry(e)):
with sentry_sdk.push_scope() as scope:
scope.set_tag("action", "retried")
sentry_sdk.capture_exception()
# ~68 hr at 11 retries
statsd.incr(f"{self.name}.retries.{self.request.retries}")
statsd.incr(f"news.tasks.retries.{self.request.retries}")
raise self.retry(
countdown=exponential_backoff(self.request.retries))
except self.MaxRetriesExceededError:
statsd.incr(self.name + ".retry_max")
statsd.incr("news.tasks.retry_max_total")
# don't bubble certain errors
if ignore_error_post_retry(e):
return
sentry_sdk.capture_exception()
def from_upload(cls,
upload,
addon,
selected_apps,
channel,
parsed_data=None):
"""
Create a Version instance and corresponding File(s) from a
FileUpload, an Addon, a list of compatible app ids, a channel id and
the parsed_data generated by parse_addon().
Note that it's the caller's responsability to ensure the file is valid.
We can't check for that here because an admin may have overridden the
validation results.
"""
assert parsed_data is not None
from olympia.addons.models import AddonFeatureCompatibility
if addon.status == amo.STATUS_DISABLED:
raise VersionCreateError(
'Addon is Mozilla Disabled; no new versions are allowed.')
license_id = None
if channel == amo.RELEASE_CHANNEL_LISTED:
previous_version = addon.find_latest_version(channel=channel,
exclude=())
if previous_version and previous_version.license_id:
license_id = previous_version.license_id
approvalnotes = None
if parsed_data.get('is_mozilla_signed_extension'):
approvalnotes = (u'This version has been signed with '
u'Mozilla internal certificate.')
version = cls.objects.create(
addon=addon,
approvalnotes=approvalnotes,
version=parsed_data['version'],
license_id=license_id,
channel=channel,
)
log.info('New version: %r (%s) from %r' %
(version, version.id, upload))
activity.log_create(amo.LOG.ADD_VERSION, version, addon)
# Update the add-on e10s compatibility since we're creating a new
# version that may change that.
e10s_compatibility = parsed_data.get('e10s_compatibility')
if e10s_compatibility is not None:
feature_compatibility = (
AddonFeatureCompatibility.objects.get_or_create(
addon=addon)[0])
feature_compatibility.update(e10s=e10s_compatibility)
compatible_apps = {}
for app in parsed_data.get('apps', []):
if app.id not in selected_apps:
# If the user chose to explicitly deselect Firefox for Android
# we're not creating the respective `ApplicationsVersions`
# which will have this add-on then be listed only for
# Firefox specifically.
continue
compatible_apps[app.appdata] = ApplicationsVersions(
version=version, min=app.min, max=app.max, application=app.id)
compatible_apps[app.appdata].save()
# See #2828: sometimes when we generate the filename(s) below, in
# File.from_upload(), cache-machine is confused and has trouble
# fetching the ApplicationsVersions that were just created. To work
# around this we pre-generate version.compatible_apps and avoid the
# queries completely.
version._compatible_apps = compatible_apps
# For backwards compatibility. We removed specific platform
# support during submission but we don't handle it any different
# beyond that yet. That means, we're going to simply set it
# to `PLATFORM_ALL` and also have the backend create separate
# files for each platform. Cleaning that up is another step.
# Given the timing on this, we don't care about updates to legacy
# add-ons as well.
# Create relevant file and update the all_files cached property on the
# Version, because we might need it afterwards.
version.all_files = [
File.from_upload(upload=upload,
version=version,
platform=amo.PLATFORM_ALL.id,
parsed_data=parsed_data)
]
version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW])
version.disable_old_files()
# After the upload has been copied to all platforms, remove the upload.
storage.delete(upload.path)
version_uploaded.send(sender=version)
if waffle.switch_is_active('enable-uploads-commit-to-git-storage'):
# Extract into git repository
AddonGitRepository.extract_and_commit_from_file_obj(
file_obj=version.all_files[0],
channel=channel,
author=upload.user)
# Generate a preview and icon for listed static themes
if (addon.type == amo.ADDON_STATICTHEME
and channel == amo.RELEASE_CHANNEL_LISTED):
dst_root = os.path.join(user_media_path('addons'), str(addon.id))
theme_data = parsed_data.get('theme', {})
version_root = os.path.join(dst_root, unicode(version.id))
utils.extract_header_img(version.all_files[0].file_path,
theme_data, version_root)
generate_static_theme_preview(theme_data, version_root, version.pk)
# Track the time it took from first upload through validation
# (and whatever else) until a version was created.
upload_start = utc_millesecs_from_epoch(upload.created)
now = datetime.datetime.now()
now_ts = utc_millesecs_from_epoch(now)
upload_time = now_ts - upload_start
log.info('Time for version {version} creation from upload: {delta}; '
'created={created}; now={now}'.format(delta=upload_time,
version=version,
created=upload.created,
now=now))
statsd.timing('devhub.version_created_from_upload', upload_time)
return version
try:
p = res.groups()[1]
if 'localepicker.properties' not in p:
p = os.path.join(p, 'localepicker.properties')
res = zip.extract_from_manifest(p)
except (zipfile.BadZipfile, IOError), e:
log.error('Error unzipping: %s, %s in file: %s' % (p, e, self.pk))
return ''
except (ValueError, KeyError), e:
log.error('No file named: %s in file: %s' % (e, self.pk))
return ''
end = time.time() - start
log.info('Extracted localepicker file: %s in %.2fs' % (self.pk, end))
statsd.timing('files.extract.localepicker', (end * 1000))
return res
@receiver(models.signals.post_save,
sender=File,
dispatch_uid='cache_localpicker')
def cache_localepicker(sender, instance, **kw):
if kw.get('raw') or not kw.get('created'):
return
try:
addon = instance.version.addon
except models.ObjectDoesNotExist:
return
def process_response(self, request, response):
statsd.incr('django.response_codes.%s' % re.sub('\d{2}$','xx', str(response.status_code)))
if hasattr(request, '_start_time'):
ms = int((time() - request._start_time) * 1000)
statsd.timing('django.response', ms)
return response
def from_upload(cls,
upload,
addon,
platforms,
channel,
source=None,
parsed_data=None):
"""
Create a Version instance and corresponding File(s) from a
FileUpload, an Addon, a list of platform ids, a channel id and the
parsed_data generated by parse_addon().
Note that it's the caller's responsability to ensure the file is valid.
We can't check for that here because an admin may have overridden the
validation results.
"""
assert parsed_data is not None
from olympia.addons.models import AddonFeatureCompatibility
if addon.status == amo.STATUS_DISABLED:
raise VersionCreateError(
'Addon is Mozilla Disabled; no new versions are allowed.')
license_id = None
if channel == amo.RELEASE_CHANNEL_LISTED:
previous_version = addon.find_latest_version(channel=channel,
exclude=())
if previous_version and previous_version.license_id:
license_id = previous_version.license_id
version = cls.objects.create(
addon=addon,
version=parsed_data['version'],
license_id=license_id,
source=source,
channel=channel,
)
log.info('New version: %r (%s) from %r' %
(version, version.id, upload))
activity.log_create(amo.LOG.ADD_VERSION, version, addon)
# Update the add-on e10s compatibility since we're creating a new
# version that may change that.
e10s_compatibility = parsed_data.get('e10s_compatibility')
if e10s_compatibility is not None:
feature_compatibility = (
AddonFeatureCompatibility.objects.get_or_create(
addon=addon)[0])
feature_compatibility.update(e10s=e10s_compatibility)
compatible_apps = {}
for app in parsed_data.get('apps', []):
compatible_apps[app.appdata] = ApplicationsVersions(
version=version, min=app.min, max=app.max, application=app.id)
compatible_apps[app.appdata].save()
# See #2828: sometimes when we generate the filename(s) below, in
# File.from_upload(), cache-machine is confused and has trouble
# fetching the ApplicationsVersions that were just created. To work
# around this we pre-generate version.compatible_apps and avoid the
# queries completely.
version._compatible_apps = compatible_apps
if addon.type in [amo.ADDON_SEARCH, amo.ADDON_STATICTHEME]:
# Search extensions and static themes are always for all platforms.
platforms = [amo.PLATFORM_ALL.id]
else:
platforms = cls._make_safe_platform_files(platforms)
# Create as many files as we have platforms. Update the all_files
# cached property on the Version while we're at it, because we might
# need it afterwards.
version.all_files = [
File.from_upload(upload,
version,
platform,
parsed_data=parsed_data) for platform in platforms
]
version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW])
version.disable_old_files()
# After the upload has been copied to all platforms, remove the upload.
storage.delete(upload.path)
version_uploaded.send(sender=version)
# Generate a preview and icon for listed static themes
if (addon.type == amo.ADDON_STATICTHEME
and channel == amo.RELEASE_CHANNEL_LISTED):
dst_root = os.path.join(user_media_path('addons'), str(addon.id))
theme_data = parsed_data.get('theme', {})
version_root = os.path.join(dst_root, unicode(version.id))
utils.extract_header_img(version.all_files[0].file_path,
theme_data, version_root)
preview = VersionPreview.objects.create(version=version)
generate_static_theme_preview(theme_data, version_root, preview.pk)
# Track the time it took from first upload through validation
# (and whatever else) until a version was created.
upload_start = utc_millesecs_from_epoch(upload.created)
now = datetime.datetime.now()
now_ts = utc_millesecs_from_epoch(now)
upload_time = now_ts - upload_start
log.info('Time for version {version} creation from upload: {delta}; '
'created={created}; now={now}'.format(delta=upload_time,
version=version,
created=upload.created,
now=now))
statsd.timing('devhub.version_created_from_upload', upload_time)
return version
def from_upload(cls,
upload,
addon,
platforms,
send_signal=True,
source=None,
is_beta=False):
from olympia.addons.models import AddonFeatureCompatibility
data = utils.parse_addon(upload, addon)
try:
license = addon.versions.latest().license_id
except Version.DoesNotExist:
license = None
v = cls.objects.create(addon=addon,
version=data['version'],
license_id=license,
source=source)
log.info('New version: %r (%s) from %r' % (v, v.id, upload))
# Update the add-on e10s compatibility since we're creating a new
# version that may change that.
e10s_compatibility = data.get('e10s_compatibility')
if e10s_compatibility is not None:
feature_compatibility = (
AddonFeatureCompatibility.objects.get_or_create(
addon=addon)[0])
feature_compatibility.update(e10s=e10s_compatibility)
AV = ApplicationsVersions
for app in data.get('apps', []):
AV(version=v, min=app.min, max=app.max, application=app.id).save()
if addon.type == amo.ADDON_SEARCH:
# Search extensions are always for all platforms.
platforms = [amo.PLATFORM_ALL.id]
else:
platforms = cls._make_safe_platform_files(platforms)
for platform in platforms:
File.from_upload(upload,
v,
platform,
parse_data=data,
is_beta=is_beta)
v.disable_old_files()
# After the upload has been copied to all platforms, remove the upload.
storage.delete(upload.path)
if send_signal:
version_uploaded.send(sender=v)
# Track the time it took from first upload through validation
# (and whatever else) until a version was created.
upload_start = utc_millesecs_from_epoch(upload.created)
now = datetime.datetime.now()
now_ts = utc_millesecs_from_epoch(now)
upload_time = now_ts - upload_start
log.info('Time for version {version} creation from upload: {delta}; '
'created={created}; now={now}'.format(delta=upload_time,
version=v,
created=upload.created,
now=now))
statsd.timing('devhub.version_created_from_upload', upload_time)
return v
def timing(key, value):
return statsd.timing(_get_key(key), value)
def timing(key, value):
# TODO(dcramer): implement timing for tsdb
return statsd.timing(_get_key(key), value,
rate=settings.SENTRY_METRICS_SAMPLE_RATE)
def log_interval(self, label):
now = datetime.datetime.utcnow()
statsd.timing(self.prefix + label, now - self._timestamp)
log.info(
"%s: %s", self.prefix + label, now - self._timestamp)
self._timestamp = now
def handle(self, *args, **kw):
statsd.timing(kw.get('key'), time.time())
