def _send_to_graphite_given_values(graphite_host, graphite_name, values, verbose=False): """Actually do the sending to graphite once values are computed. values is either a list of (time_t, value) pairs, or a dict from module-name to a time_t/value list. Returns the latest time_t seen. """ if not values: return # 'values' is either a list of (time_t, value) pairs, or a # map from module_name to list of (time_t, value) pairs. if isinstance(values, dict): assert '%(module_name)s' in graphite_name, graphite_name retval = 0 for (module_name, module_values) in values.iteritems(): if module_name is None: # Holds sum over all modules key_suffix = graphite_name.replace('%(module_name)s.', '') else: module = module_name.replace('-', '_') + '_module' key_suffix = graphite_name.replace('%(module_name)s', module) retval = max( retval, _send_to_graphite_given_values(graphite_host, key_suffix, module_values, verbose)) return retval else: assert '%(module_name)s' not in graphite_name, graphite_name key = 'webapp.gae.dashboard.%s' % graphite_name graphite_data = [] for (time_t, value) in values: if hasattr(value, '__iter__'): # a list or the like assert '%(percentile)s' in graphite_name, graphite_name percentile_map = _get_percentiles(value) for (pct, pct_value) in percentile_map.iteritems(): pct_label = 'pct%d' % pct pct_key = key.replace('%(percentile)s', pct_label) graphite_data.append((pct_key, (time_t, pct_value))) else: assert '%(percentile)s' not in graphite_name, graphite_name graphite_data.append((key, (time_t, value))) graphite_data.sort() if verbose: if graphite_host: print "--> Sending to graphite:" else: print "--> Would send to graphite:" # Replace the common prefix with an abbreviation to make it # more likely each line will fit within 80 chars. print '\n'.join( str(v).replace('webapp.gae.dashboard.summary.', 'w.g.d.s.') for v in graphite_data) graphite_util.send_to_graphite(graphite_host, graphite_data) if not graphite_data: return 0 else: return max(time_t for (_, (time_t, __)) in graphite_data)
def _send_to_graphite_given_values(graphite_host, graphite_name, values, verbose=False): """Actually do the sending to graphite once values are computed. values is either a list of (time_t, value) pairs, or a dict from module-name to a time_t/value list. Returns the latest time_t seen. """ if not values: return # 'values' is either a list of (time_t, value) pairs, or a # map from module_name to list of (time_t, value) pairs. if isinstance(values, dict): assert '%(module_name)s' in graphite_name, graphite_name retval = 0 for (module_name, module_values) in values.iteritems(): if module_name is None: # Holds sum over all modules key_suffix = graphite_name.replace('%(module_name)s.', '') else: module = module_name.replace('-', '_') + '_module' key_suffix = graphite_name.replace('%(module_name)s', module) retval = max(retval, _send_to_graphite_given_values( graphite_host, key_suffix, module_values, verbose)) return retval else: assert '%(module_name)s' not in graphite_name, graphite_name key = 'webapp.gae.dashboard.%s' % graphite_name graphite_data = [] for (time_t, value) in values: if hasattr(value, '__iter__'): # a list or the like assert '%(percentile)s' in graphite_name, graphite_name percentile_map = _get_percentiles(value) for (pct, pct_value) in percentile_map.iteritems(): pct_label = 'pct%d' % pct pct_key = key.replace('%(percentile)s', pct_label) graphite_data.append((pct_key, (time_t, pct_value))) else: assert '%(percentile)s' not in graphite_name, graphite_name graphite_data.append((key, (time_t, value))) graphite_data.sort() if verbose: if graphite_host: print "--> Sending to graphite:" else: print "--> Would send to graphite:" # Replace the common prefix with an abbreviation to make it # more likely each line will fit within 80 chars. print '\n'.join( str(v).replace('webapp.gae.dashboard.summary.', 'w.g.d.s.') for v in graphite_data) graphite_util.send_to_graphite(graphite_host, graphite_data) if not graphite_data: return 0 else: return max(time_t for (_, (time_t, __)) in graphite_data)
def main(graphite_host, month=None, year=None, verbose=False, dry_run=False): csv_file_contents = get_csv_contents_from_s3(month=month, year=year) graphite_data = collate_data(csv_file_contents) if dry_run: graphite_host = None # so we don't actually send to graphite verbose = True # print what we would have done if verbose: if graphite_host: print "--> Sending to graphite:" else: print "--> Would send to graphite:" print '\n'.join(str(v) for v in graphite_data) if not dry_run: graphite_util.send_to_graphite(graphite_host, graphite_data)
def report_out_of_memory_errors(yyyymmdd_hh, graphite_host): # This bigquery query is modified from email_bq_data.py query = """\ SELECT module_id, integer(app_logs.time) as time_t FROM [logs_hourly.requestlogs_%s] WHERE app_logs.message CONTAINS 'Exceeded soft private memory limit' AND module_id IS NOT NULL """ % (yyyymmdd_hh) data = bq_util.query_bigquery(query) records = [('webapp.%(module_id)s_module.stats.oom' % row, (row['time_t'], 1)) for row in data] if not graphite_host: print 'Would send to graphite: %s' % records else: graphite_util.send_to_graphite(graphite_host, records)
def main(graphite_host, date, verbose=False, dry_run=False): project_to_cost_so_far = get_bigquery_costs(date) graphite_data = format_for_graphite(project_to_cost_so_far) if dry_run: graphite_host = None # so we don't actually send to graphite verbose = True # print what we would have done if verbose: if graphite_host: print "--> Sending to graphite:" else: print "--> Would send to graphite:" print graphite_data if not dry_run: graphite_util.send_to_graphite(graphite_host, graphite_data)
def get_and_parse_usage_info(service, date_string, graphite_host, dry_run=False, verbose=False): usage_info = _get_usage_info(service, date_string) graphite_data = [] for record in usage_info: graphite_data.extend(_parse_one_record(record)) if dry_run: graphite_host = None # so we don't actually send to graphite verbose = True # print what we would have done if verbose: if graphite_host: print "--> Sending to graphite:" else: print "--> Would send to graphite:" print "\n".join(str(v) for v in graphite_data) graphite_util.send_to_graphite(graphite_host, graphite_data)
def main(graphite_host, start_time_t, end_time_t, verbose=False, dry_run=False): raw_data = get_google_services_costs(start_time_t, end_time_t) graphite_data = format_for_graphite(raw_data, end_time_t) if dry_run: graphite_host = None # so we don't actually send to graphite verbose = True # print what we would have done if verbose: if graphite_host: print "--> Sending to graphite:" else: print "--> Would send to graphite:" print '\n'.join([str(x) for x in graphite_data]) if not dry_run: graphite_util.send_to_graphite(graphite_host, graphite_data)
def get_and_parse_usage_info(service, date_string, graphite_host, dry_run=False, verbose=False): usage_info = _get_usage_info(service, date_string) graphite_data = [] for record in usage_info: graphite_data.extend(_parse_one_record(record)) if dry_run: graphite_host = None # so we don't actually send to graphite verbose = True # print what we would have done if verbose: if graphite_host: print "--> Sending to graphite:" else: print "--> Would send to graphite:" print '\n'.join(str(v) for v in graphite_data) graphite_util.send_to_graphite(graphite_host, graphite_data)