def main():
"""Entry point when called on the command-line.
"""
# Locale
locale.setlocale(locale.LC_ALL, "")
parser = argparse.ArgumentParser(description="api_stats.list prints the keys available in the data")
parser.add_argument("-v", "--verbose", action="count", default=1, dest="verbosity", help="augments verbosity level")
parser.add_argument("data", help="file from which to read the data")
args = parser.parse_args()
setup_logging(args.verbosity)
keys = {}
if PY3:
fp = open(args.data, "r", encoding="utf-8")
else:
fp = open(args.data, "rb")
try:
nb_lines = 0
for line in fp:
if not line:
continue
nb_lines += 1
data = json.loads(line)
if data["format"] != 1:
logger.critical("Found line in unknown format %r", data["format"])
sys.exit(1)
for key, value in iteritems(data["values"]):
key = key.split("/")
pos = keys
while key:
pos = pos.setdefault(key.pop(0), {})
logger.info("Read %d lines", nb_lines)
finally:
fp.close()
print_tree(keys)
def process_configuration(data_output, configuration_file):
"""Process a configuration file and update the data.
"""
stats = StatisticsRecorder()
env = {'stats': stats}
try:
execfile(configuration_file, env, env)
except IOError as e:
logger.critical("Cannot open configuration file: %s", e)
sys.exit(1)
logger.info("Recorded %d values", len(stats.recorded_values))
if data_output is None:
logger.warning("Discarding output as requested")
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Output was:")
for key, value in iteritems(stats.recorded_values):
logger.debug("%s = %r", key, value)
else:
if PY3:
fp = open(data_output, 'a', encoding='utf-8', newline='\n')
else:
fp = open(data_output, 'ab')
try:
json.dump({'format': 1,
'date': datetime.utcnow().isoformat(),
'values': stats.recorded_values},
fp,
ensure_ascii=True,
indent=None,
sort_keys=True)
fp.write('\n')
finally:
fp.close()
def request(self, method, url, **kwargs):
logger.info("%s %s", method, url)
return requests.request(method, url, **kwargs)
def get_json(self, url, **kwargs):
logger.info("Downloading JSON from %s", url)
return requests.get(url, **kwargs).json()
def main():
"""Entry point when called on the command-line.
"""
# Locale
locale.setlocale(locale.LC_ALL, '')
parser = argparse.ArgumentParser(
description="api_stats.plot displays the recorded data using "
"matplotlib")
parser.add_argument('-v', '--verbose', action='count', default=1,
dest='verbosity',
help="augments verbosity level")
parser.add_argument('-m', action='append', dest='maps', nargs=2,
default=[], help="map input data to a plot")
parser.add_argument('-o', '--output', action='store_true', default=False,
help="Output to PNG files")
parser.add_argument('data', help="file from which to read the data")
args = parser.parse_args()
setup_logging(args.verbosity)
# Compile mappings
maps = [(re.compile(r_from), r_to) for r_from, r_to in args.maps]
lines = {} # dict(key:str, dict(date:str, value:float))
if PY3:
fp = open(args.data, 'r', encoding='utf-8')
else:
fp = open(args.data, 'rb')
try:
nb_lines = 0
for line in fp:
if not line:
continue
nb_lines += 1
data = json.loads(line)
if data['format'] != 1:
logger.critical("Found line in unknown format %r",
data['format'])
sys.exit(1)
date = data['date']
for key, value in iteritems(data['values']):
for r_from, r_to in maps:
new_key, subs = r_from.subn(r_to, key)
if subs:
logger.debug("MAP: %s -> %s", key, new_key)
assert isinstance(value, (long, int, float))
this_line = lines.setdefault(new_key, {})
this_line[date] = this_line.get(date, 0) + value
logger.info("Read %d lines", nb_lines)
finally:
fp.close()
if args.output:
matplotlib.use('Agg')
# dict(title:str, dict(key:tuple, (dict(date:str, value:float),
# options:str)))
plots = {}
for key, values in iteritems(lines):
options = '-'
if '^' in key:
key, options = key.split('^', 1)
key = key.split(':')
if len(key) > 1:
title = key[0]
key = key[1:]
else:
title = None
plots.setdefault(title, {})[tuple(key)] = values, options
import matplotlib.pyplot as plt
for title, plot in iteritems(plots):
logger.info("Creating figure %s", title)
fig = plt.figure()
graph = fig.add_subplot(111)
plt.title(title)
for key, (line, options) in iteritems(plot):
label = ':'.join(key)
logger.info("Plotting %s", label)
x = []
y = []
for k, v in sorted((date2num(parse_date(k)), v)
for k, v in iteritems(line)):
x.append(k)
y.append(v)
graph.plot(x, y, options, label=label)
# graph.xaxis.set_major_locator(MonthLocator())
# graph.xaxis.set_minor_locator(DayLocator())
graph.xaxis.set_major_formatter(DateFormatter("%m-%d"))
graph.legend(loc=2)
graph.set_ylim(bottom=0)
fig.savefig('%s.png' % title)
if not args.output:
plt.show()