def _run_one(self, job_class, config, force=False):
_debug = self.config.logger.debug
seconds = convert_frequency(config.frequency)
time_ = config.time
if not force:
if not self.time_to_run(job_class, time_):
_debug("skipping %r because it's not time to run", job_class)
return
ok, dependency_error = self.check_dependencies(job_class)
if not ok:
_debug("skipping %r dependencies aren't met [%s]", job_class,
dependency_error)
return
_debug('about to run %r', job_class)
app_name = job_class.app_name
info = self.job_state_database.get(app_name)
last_success = None
now = utc_now()
log_run = True
try:
t0 = time.time()
for last_success in self._run_job(job_class, config, info):
t1 = time.time()
_debug('successfully ran %r on %s', job_class, last_success)
self._remember_success(job_class, last_success, t1 - t0)
# _run_job() returns a generator, so we don't know how
# many times this will loop. Anyway, we need to reset the
# 't0' for the next loop if there is one.
t0 = time.time()
exc_type = exc_value = exc_tb = None
except (OngoingJobError, RowLevelLockError):
# It's not an actual runtime error. It just basically means
# you can't start crontabber right now.
log_run = False
raise
except Exception:
t1 = time.time()
exc_type, exc_value, exc_tb = sys.exc_info()
if self.config.sentry and self.config.sentry.dsn:
client = raven_client.get_client(self.config.sentry.dsn)
identifier = client.get_ident(client.captureException())
self.config.logger.info(
'Error captured in Sentry. Reference: %s' % identifier)
_debug('error when running %r on %s',
job_class,
last_success,
exc_info=True)
self._remember_failure(job_class, t1 - t0, exc_type, exc_value,
exc_tb)
finally:
if log_run:
self._log_run(job_class, seconds, time_, last_success, now,
exc_type, exc_value, exc_tb)
def sentrytest(self):
"""return true if we managed to send a sample raven exception"""
if not (self.config.sentry and self.config.sentry.dsn):
raise SentryConfigurationError('sentry dsn not configured')
client = raven_client.get_client(self.config.sentry.dsn)
identifier = client.captureMessage('Sentry test sent from crontabber')
self.logger.info('Sentry successful identifier: %s', identifier)
return True
def alert_sentry(self, dsn, vulnerabilities):
client = raven_client.get_client(dsn)
client.context.activate()
client.context.merge({
'extra': {
'data': {vuln.key: vuln.summary for vuln in vulnerabilities},
},
})
client.captureMessage('Dependency security check failed')
def sentrytest(self):
"""return true if we managed to send a sample raven exception"""
if not (self.config.sentry and self.config.sentry.dsn):
raise SentryConfigurationError('sentry dsn not configured')
client = raven_client.get_client(self.config.sentry.dsn)
identifier = client.captureMessage('Sentry test sent from crontabber')
self.logger.info('Sentry successful identifier: %s', identifier)
return True
def alert_sentry(self, dsn, vulnerabilities):
client = raven_client.get_client(dsn)
client.context.activate()
client.context.merge({
'extra': {
'data': {vuln.key: vuln.summary
for vuln in vulnerabilities},
},
})
client.captureMessage('Dependency security check failed')
def _send_to_sentry(self, tag, func, *args, **kwargs):
"""Execute this when an exception has happened only.
If self.config.sentry.dsn is set up, it will try to send it
to Sentry. If not configured, nothing happens.
"""
try:
dsn = self.config.sentry.dsn
except KeyError:
# if self.config is not a DotDict, we can't access the sentry.dsn
dsn = None
if dsn:
extra = {
'class': self.__class__.__name__,
'tag': tag, # this can 'predicate' or 'action'
}
args = inspect.getcallargs(func, *args, **kwargs)['args']
# For every crash acted on, it's always
# act(raw_crash, ...) etc.
# But be defensive in case the first argument isn't there,
# isn't a dict or doesn't have a 'uuid'.
if args and isinstance(args[0], collections.Mapping):
crash_id = args[0].get('uuid')
if crash_id:
extra['crash_id'] = crash_id
try:
client = raven_client.get_client(dsn)
client.context.activate()
client.context.merge({'extra': extra})
try:
identifier = client.captureException()
self.config.logger.info(
'Error captured in Sentry! '
'Reference: {}'.format(
identifier
)
)
return True # it worked!
finally:
client.context.clear()
except Exception:
self.config.logger.error(
'Unable to report error with Raven',
exc_info=True,
)
else:
self.config.logger.warning(
'Raven DSN is not configured and an exception happened'
)
def _capture_error(self, crash_id, exc_type, exc_value, exc_tb):
"""Capture an error in sentry if able
The `exc_*` arguments come from calling `sys.exc_info`.
:arg crash_id: a crash id
:arg exc_type: the exc class
:arg exc_value: the exception
:arg exc_tb: the traceback for the exception
"""
if self.config.sentry and self.config.sentry.dsn:
try:
if isinstance(exc_value, collections.Sequence):
# Then it's already an iterable!
exceptions = exc_value
else:
exceptions = [exc_value]
client = raven_client.get_client(self.config.sentry.dsn)
client.context.activate()
client.context.merge({'extra': {
'crash_id': crash_id,
}})
try:
for exception in exceptions:
identifier = client.captureException(exception)
self.config.logger.info(
'Error captured in Sentry! Reference: {}'.format(
identifier))
finally:
client.context.clear()
except Exception:
self.config.logger.error('Unable to report error with Raven',
exc_info=True)
else:
self.config.logger.warning(
'Sentry DSN is not configured and an exception happened')
def _capture_error(self, crash_id, exc_type, exc_value, exc_tb):
"""Capture an error in sentry if able
The `exc_*` arguments come from calling `sys.exc_info`.
:arg crash_id: a crash id
:arg exc_type: the exc class
:arg exc_value: the exception
:arg exc_tb: the traceback for the exception
"""
if self.config.sentry and self.config.sentry.dsn:
try:
if isinstance(exc_value, collections.Sequence):
# Then it's already an iterable!
exceptions = exc_value
else:
exceptions = [exc_value]
client = raven_client.get_client(self.config.sentry.dsn)
client.context.activate()
client.context.merge({'extra': {
'crash_id': crash_id,
}})
try:
for exception in exceptions:
identifier = client.captureException(exception)
self.config.logger.info(
'Error captured in Sentry! Reference: {}'.format(identifier)
)
finally:
client.context.clear()
except Exception:
self.config.logger.error('Unable to report error with Raven', exc_info=True)
else:
self.config.logger.warning('Sentry DSN is not configured and an exception happened')
def _transform(self, crash_id):
"""this implementation is the framework on how a raw crash is
converted into a processed crash. The 'crash_id' passed in is used as
a key to fetch the raw crash from the 'source', the conversion funtion
implemented by the 'processor_class' is applied, the
processed crash is saved to the 'destination'"""
try:
raw_crash = self.source.get_raw_crash(crash_id)
dumps = self.source.get_raw_dumps_as_files(crash_id)
except CrashIDNotFound:
self.processor.reject_raw_crash(
crash_id,
'this crash cannot be found in raw crash storage'
)
return
except Exception as x:
self.config.logger.warning(
'error loading crash %s',
crash_id,
exc_info=True
)
self.processor.reject_raw_crash(
crash_id,
'error in loading: %s' % x
)
return
try:
processed_crash = self.source.get_unredacted_processed(
crash_id
)
except CrashIDNotFound:
processed_crash = DotDict()
try:
if 'uuid' not in raw_crash:
raw_crash.uuid = crash_id
processed_crash = (
self.processor.process_crash(
raw_crash,
dumps,
processed_crash,
)
)
""" bug 866973 - save_raw_and_processed() instead of just
save_processed(). The raw crash may have been modified
by the processor rules. The individual crash storage
implementations may choose to honor re-saving the raw_crash
or not.
"""
self.destination.save_raw_and_processed(
raw_crash,
None,
processed_crash,
crash_id
)
self.config.logger.info('saved - %s', crash_id)
except Exception as exception:
# Immediately capture this as local variables.
# During this error handling we're going to be using other
# try:except: constructs (e.g. swallowing raven send errors)
# so we can't reference `sys.exc_info()` later.
exc_type, exc_value, exc_tb = sys.exc_info()
if self.config.sentry and self.config.sentry.dsn:
try:
if isinstance(exception, collections.Sequence):
# Then it's already an iterable!
exceptions = exception
else:
exceptions = [exception]
client = raven_client.get_client(self.config.sentry.dsn)
client.context.activate()
client.context.merge({'extra': {
'crash_id': crash_id,
}})
try:
for exception in exceptions:
identifier = client.captureException(
exception
)
self.config.logger.info(
'Error captured in Sentry! '
'Reference: {}'.format(
identifier
)
)
finally:
client.context.clear()
except Exception:
self.config.logger.error(
'Unable to report error with Raven',
exc_info=True,
)
else:
self.config.logger.warning(
'Raven DSN is not configured and an exception happened'
)
# Why not just do `raise exception`?
# Because if we don't do it this way, the eventual traceback
# is going to point to *this* line (right after this comment)
# rather than the actual error where it originally happened.
raise exc_type, exc_value, exc_tb
finally:
# earlier, we created the dumps as files on the file system,
# we need to clean up after ourselves.
for a_dump_pathname in dumps.itervalues():
try:
if "TEMPORARY" in a_dump_pathname:
os.unlink(a_dump_pathname)
except OSError as x:
# the file does not actually exist
self.config.logger.info(
'deletion of dump failed: %s',
x,
)
def _transform(self, crash_id):
"""this implementation is the framework on how a raw crash is
converted into a processed crash. The 'crash_id' passed in is used as
a key to fetch the raw crash from the 'source', the conversion funtion
implemented by the 'processor_class' is applied, the
processed crash is saved to the 'destination'"""
try:
raw_crash = self.source.get_raw_crash(crash_id)
dumps = self.source.get_raw_dumps_as_files(crash_id)
except CrashIDNotFound:
self.processor.reject_raw_crash(
crash_id, 'this crash cannot be found in raw crash storage')
return
except Exception as x:
self.config.logger.warning('error loading crash %s',
crash_id,
exc_info=True)
self.processor.reject_raw_crash(crash_id,
'error in loading: %s' % x)
return
try:
processed_crash = self.source.get_unredacted_processed(crash_id)
except CrashIDNotFound:
processed_crash = DotDict()
try:
if 'uuid' not in raw_crash:
raw_crash.uuid = crash_id
processed_crash = (self.processor.process_crash(
raw_crash,
dumps,
processed_crash,
))
""" bug 866973 - save_raw_and_processed() instead of just
save_processed(). The raw crash may have been modified
by the processor rules. The individual crash storage
implementations may choose to honor re-saving the raw_crash
or not.
"""
self.destination.save_raw_and_processed(raw_crash, None,
processed_crash, crash_id)
self.config.logger.info('saved - %s', crash_id)
except Exception as exception:
# Immediately capture this as local variables.
# During this error handling we're going to be using other
# try:except: constructs (e.g. swallowing raven send errors)
# so we can't reference `sys.exc_info()` later.
exc_type, exc_value, exc_tb = sys.exc_info()
if self.config.sentry and self.config.sentry.dsn:
try:
if isinstance(exception, collections.Sequence):
# Then it's already an iterable!
exceptions = exception
else:
exceptions = [exception]
client = raven_client.get_client(self.config.sentry.dsn)
client.context.activate()
client.context.merge({'extra': {
'crash_id': crash_id,
}})
try:
for exception in exceptions:
identifier = client.captureException(exception)
self.config.logger.info(
'Error captured in Sentry! '
'Reference: {}'.format(identifier))
finally:
client.context.clear()
except Exception:
self.config.logger.error(
'Unable to report error with Raven',
exc_info=True,
)
else:
self.config.logger.warning(
'Raven DSN is not configured and an exception happened')
# Why not just do `raise exception`?
# Because if we don't do it this way, the eventual traceback
# is going to point to *this* line (right after this comment)
# rather than the actual error where it originally happened.
raise exc_type, exc_value, exc_tb
finally:
# earlier, we created the dumps as files on the file system,
# we need to clean up after ourselves.
for a_dump_pathname in dumps.itervalues():
try:
if "TEMPORARY" in a_dump_pathname:
os.unlink(a_dump_pathname)
except OSError as x:
# the file does not actually exist
self.config.logger.info(
'deletion of dump failed: %s',
x,
)
def _run_one(self, job_class, config, force=False):
seconds = convert_frequency(config.frequency)
time_ = config.time
if not force:
if not self.time_to_run(job_class, time_):
self.logger.debug("skipping %r because it's not time to run", job_class)
return
self.logger.debug('about to run %r', job_class)
app_name = job_class.app_name
info = self.job_state_database.get(app_name)
last_success = None
now = utc_now()
log_run = True
exc_type = exc_value = exc_tb = None
try:
t0 = time.time()
for last_success in self._run_job(job_class, config, info):
t1 = time.time()
self.logger.debug('successfully ran %r on %s', job_class, last_success)
self._remember_success(job_class, last_success, t1 - t0)
# _run_job() returns a generator, so we don't know how
# many times this will loop. Anyway, we need to reset the
# 't0' for the next loop if there is one.
t0 = time.time()
except (OngoingJobError, RowLevelLockError):
# It's not an actual runtime error. It just basically means
# you can't start crontabber right now.
log_run = False
raise
except Exception:
t1 = time.time()
exc_type, exc_value, exc_tb = sys.exc_info()
if self.config.sentry and self.config.sentry.dsn:
client = raven_client.get_client(self.config.sentry.dsn)
identifier = client.get_ident(client.captureException())
self.logger.info('Error captured in Sentry. Reference: %s' % identifier)
self.logger.debug(
'error when running %r on %s', job_class, last_success, exc_info=True
)
self._remember_failure(
job_class,
t1 - t0,
exc_type,
exc_value,
exc_tb
)
finally:
if log_run:
self._log_run(
job_class,
seconds,
time_,
last_success,
now,
exc_type, exc_value, exc_tb
)