def __init__(self, app_id, lookback_minutes, new_relic_account,
new_relic_app_id, event_type):
super(ErrorsByVersionProcessor,
self).__init__(app_id, lookback_minutes, new_relic_account,
new_relic_app_id, event_type)
self._to_save = []
errors_with_details = self.get_errors_with_details(lookback_minutes)
events = []
cumulative_errors_found = 0
cumulative_previously_known = 0
for cr_error in errors_with_details:
todays_date = cr_error.current_date().strftime('%Y-%m-%d')
e = Event(new_relic_account, new_relic_app_id, event_type,
datetime.now())
for k, v in cr_error.as_event_dict(app_id).items():
e.set(k, v)
for version, version_occurrences in cr_error.current_date_occurrences(
).items():
previously_known = CRErrorHistoryDAO.num_previously_known(
self._app_id, event_type, cr_error.crittercism_hash(),
version, todays_date)
delta = version_occurrences - previously_known
cumulative_errors_found += version_occurrences
cumulative_previously_known += previously_known
logging.getLogger().debug(
'Building event for cr_error=%s version=%s previously_known=%s num_new=%s',
cr_error.crittercism_hash(), version, previously_known,
delta)
version_event = copy.deepcopy(e)
version_event.set(u'version', version)
events += [version_event] * delta
self._to_save.append(
(event_type, cr_error.crittercism_hash(), version,
todays_date, version_occurrences))
self._events = events
logging.getLogger().info(
'AppId=%s Event=%s num_found=%s num_known=%s num_new=%s', app_id,
event_type, cumulative_errors_found, cumulative_previously_known,
len(events))
def __init__(self, app_id, lookback_minutes, new_relic_account,
new_relic_app_id, event_type):
super(MetricsAggregateSplitter,
self).__init__(app_id, lookback_minutes, new_relic_account,
new_relic_app_id, event_type)
volumes = self.get_data(self.get_volume_metric())
events_in_volume = []
self._events = []
for volume in volumes.values():
events_in_volume.append(volume.total())
num_events = max(events_in_volume)
self._start_time, self._end_time = volumes.values()[0].time_range()
logging.getLogger().debug('start_time: %s end_time: %s',
self._start_time, self._end_time)
if not num_events:
logging.getLogger().info('No events to upload.')
return
logging.getLogger().debug('Creating events')
for i in range(num_events):
self._events.append(
Event(new_relic_account, new_relic_app_id, event_type,
random_timestamp(self._start_time, self._end_time)))
logging.getLogger().debug('Extracting ranges and setting volumes.')
self._ranges = self.get_event_ranges(volumes)
logging.getLogger().debug(
'Getting the rest of the data and processing')
for metric, strategy in self.get_metrics_and_strategies():
metric_data = self.get_data(metric)
strategy(self, metric, metric_data)
logging.getLogger().info('ETL Completed')
def __init__(self, app_id, lookback_minutes, new_relic_account, new_relic_app_id, event_type):
super(MetricsAggregateSplitter, self).__init__(app_id, lookback_minutes, new_relic_account, new_relic_app_id, event_type)
volumes = self.get_data(self.get_volume_metric())
num_queried_events = volumes.values()[0].total()
self._start_time, self._end_time = volumes.values()[0].time_range()
logging.getLogger().debug('start_time: %s end_time: %s', self._start_time, self._end_time)
num_events = num_queried_events - self.get_num_already_uploaded()
if not num_events:
if num_queried_events:
logging.getLogger().info('All events queried were been uploaded. num_queried=%s', num_queried_events)
logging.getLogger().info('No new events to upload.')
self._events = []
return
self._fudge_ratio = num_events / num_queried_events
logging.getLogger().debug('Creating events')
self._events = [Event(new_relic_account, new_relic_app_id, event_type, random_timestamp(self._start_time,
self._end_time))
for
i in range(num_events)]
logging.getLogger().debug('Extracting ranges and setting volumes.')
self._ranges = self.get_event_ranges(volumes)
logging.getLogger().debug('Getting the rest of the data and processing')
for metric, strategy in self.get_metrics_and_strategies():
metric_data = self.get_data(metric)
strategy(self, metric, metric_data)
logging.getLogger().info('ETL Completed')
def __init__(self, app_id, lookback_minutes, new_relic_account, new_relic_app_id, event_type):
super(ErrorsByVersionProcessor, self).__init__(app_id, lookback_minutes,
new_relic_account, new_relic_app_id, event_type)
self._to_save = []
errors_with_details = self.get_errors_with_details(lookback_minutes)
events = []
cumulative_errors_found = 0
cumulative_previously_known = 0
for cr_error in errors_with_details:
todays_date = cr_error.current_date().strftime('%Y-%m-%d')
e = Event(new_relic_account, new_relic_app_id, event_type, datetime.now())
for k, v in cr_error.as_event_dict().items():
e.set(k, v)
for version, version_occurrences in cr_error.current_date_occurrences().items():
previously_known = CRErrorHistoryDAO.num_previously_known(self._app_id, event_type,
cr_error.crittercism_hash(),
version, todays_date)
delta = version_occurrences - previously_known
cumulative_errors_found += version_occurrences
cumulative_previously_known += previously_known
logging.getLogger().debug('Building event for cr_error=%s version=%s previously_known=%s num_new=%s',
cr_error.crittercism_hash(), version, previously_known, delta)
version_event = copy.deepcopy(e)
version_event.set(u'version', version)
events += [version_event] * delta
self._to_save.append((event_type, cr_error.crittercism_hash(), version,
todays_date, version_occurrences))
self._events = events
logging.getLogger().info('AppId=%s Event=%s num_found=%s num_known=%s num_new=%s',
app_id, event_type, cumulative_errors_found, cumulative_previously_known, len(events))
def make_event(self, trace, group_name):
dt = datetime.strptime(trace[u'traceTs'], '%Y-%m-%dT%H:%M:%S.%fZ')
e = Event(self._new_relic_account, self._new_relic_app_id, self._event_type, dt)
e.set('group', group_name)
for k, v in trace.items():
e.set(k, v)
return e
