def test_fileencoding(self):
tests = [
("foo_bar", True), ("1234.txt", True), ("xYz.DAT", True),
("foo/bar.txt", False), ("foo/bar+bas-xyz.txt", False),
(u"foo\u1234-bar\u4321", False),
("foobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobar",
u"foobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobar\u2026"
)
]
for name, expect_identical in tests:
path = common.filename_encode(name)
if expect_identical == True:
self.assertEqual(
path, name, "Encoding '%s' shouldn't change string" % name)
else:
self.assertTrue(
re.match(r"^[\w\.\-]*$", path),
"Encoding '%s' should replace all special characters" %
name)
if isinstance(expect_identical, basestring):
self.assertEqual(
common.filename_decode(path), expect_identical,
"Encoding and decoding '%s' should produce a truncated string as result"
% name)
else:
self.assertEqual(
common.filename_decode(path), name,
"Encoding and decoding '%s' should produce the original string"
% name)
def save_stats(server_type, data, factor=1):
base_dir = os.path.join(get_config().get("stats", "dataDirectory"), common.filename_encode(server_type))
for month, month_data in data.iteritems():
for name, file_data in month_data.iteritems():
path = os.path.join(base_dir, common.filename_encode(month), common.filename_encode(name + ".json"))
if os.path.exists(path):
with codecs.open(path, "rb", encoding="utf-8") as fileobj:
existing = json.load(fileobj)
else:
existing = {}
merge_objects(existing, file_data, factor)
dir = os.path.dirname(path)
try:
os.makedirs(dir)
except OSError, e:
if e.errno != errno.EEXIST:
raise
with codecs.open(path, "wb", encoding="utf-8") as fileobj:
json.dump(existing, fileobj, indent=2, sort_keys=True)
def save_stats(server_type, data, factor=1):
base_dir = os.path.join(get_config().get("stats", "dataDirectory"), common.filename_encode(server_type))
for month, month_data in data.iteritems():
for name, file_data in month_data.iteritems():
path = os.path.join(base_dir, common.filename_encode(month), common.filename_encode(name + ".json"))
if os.path.exists(path):
with codecs.open(path, "rb", encoding="utf-8") as fileobj:
existing = json.load(fileobj)
else:
existing = {}
merge_objects(existing, file_data, factor)
dir = os.path.dirname(path)
try:
os.makedirs(dir)
except OSError, e:
if e.errno != errno.EEXIST:
raise
with codecs.open(path, "wb", encoding="utf-8") as fileobj:
json.dump(existing, fileobj, indent=2, sort_keys=True)
def test_fileencoding(self):
tests = [
("foo_bar", True),
("1234.txt", True),
("xYz.DAT", True),
("foo/bar.txt", False),
("foo/bar+bas-xyz.txt", False),
(u"foo\u1234-bar\u4321", False),
]
for name, expect_identical in tests:
path = common.filename_encode(name)
if expect_identical:
self.assertEqual(path, name, "Encoding '%s' shouldn't change string" % name)
else:
self.assertTrue(re.match(r"^[\w\.\-]*$", path), "Encoding '%s' should replace all special characters" % name)
self.assertEqual(common.filename_decode(path), name, "Encoding and decoding '%s' should produce the original string" % name)
def generate_pages(datadir, outputdir):
for server_type, server_type_dir in get_names(datadir, True):
baseURL = get_config().get('stats', 'baseURL_' + server_type)
filedata = {}
current_month = None
for month, month_dir in get_names(server_type_dir, True):
if current_month == None or month > current_month:
current_month = month
for filename, path in get_names(month_dir, False):
filename = re.sub(r'\.json$', '', filename)
with codecs.open(path, 'rb', encoding='utf-8') as file:
data = json.load(file)
overview_url = '../../overview-' + common.filename_encode(filename + '.html')
filtered_urls = {}
for field in common.fields:
if field['name'] not in data:
continue
# Create filtered views for the first thirty values of a field if they
# have filtered data.
sorted_field = get_template_environment().filters['sortfield'](data[field['name']], field)
for name, value in sorted_field[0:get_default_count(field)]:
if filter(lambda k: k not in ('hits', 'bandwidth'), value.iterkeys()):
outputfile = os.path.join(outputdir,
common.filename_encode(server_type),
common.filename_encode(month),
common.filename_encode(filename),
'filtered-%s-%s.html' % (
common.filename_encode(field['name']),
common.filename_encode(name),
))
generate_file_stats(outputfile, month, baseURL + filename, overview_url,
value, filter={'field': field, 'value': name})
if not field['name'] in filtered_urls:
filtered_urls[field['name']] = {}
filtered_urls[field['name']][name] = os.path.basename(outputfile)
outputfile = os.path.join(outputdir,
common.filename_encode(server_type),
common.filename_encode(month),
common.filename_encode(filename),
'index.html')
generate_file_stats(outputfile, month, baseURL + filename, overview_url,
data, filtered_urls=filtered_urls)
if filename not in filedata:
filedata[filename] = {}
month_url = '%s/%s/%s' % (common.filename_encode(month),
common.filename_encode(filename),
'index.html')
filedata[filename][month] = {'url': month_url, 'hits': data['hits'], 'bandwidth': data['bandwidth']}
monthdata = {}
for filename, data in filedata.iteritems():
outputfile = os.path.join(outputdir,
common.filename_encode(server_type),
'overview-' + common.filename_encode(filename + '.html'))
generate_file_overview(outputfile, baseURL + filename, data)
if current_month in data:
monthdata[filename] = dict(data[current_month])
outputfile = os.path.join(outputdir, common.filename_encode(server_type), 'index.html')
generate_main_page(outputfile, current_month, baseURL, monthdata)
def generate_pages(datadir, outputdir):
for server_type, server_type_dir in get_names(datadir, True):
baseURL = get_config().get("stats", "baseURL_" + server_type)
filedata = {}
current_month = None
for month, month_dir in get_names(server_type_dir, True):
if current_month == None or month > current_month:
current_month = month
for filename, path in get_names(month_dir, False):
filename = re.sub(r"\.json$", "", filename)
with codecs.open(path, "rb", encoding="utf-8") as file:
data = simplejson.load(file)
overview_url = "../../overview-" + common.filename_encode(filename + ".html")
filtered_urls = {}
for field in common.fields:
if field["name"] not in data:
continue
# Create filtered views for the first thirty values of a field if they
# have filtered data.
for name, value in get_template_environment().filters["sortfield"](data[field["name"]], field)[0:30]:
if filter(lambda k: k not in ("hits", "bandwidth"), value.iterkeys()):
outputfile = os.path.join(outputdir,
common.filename_encode(server_type),
common.filename_encode(month),
common.filename_encode(filename),
"filtered-%s-%s.html" % (
common.filename_encode(field["name"]),
common.filename_encode(name),
))
generate_file_stats(outputfile, month, baseURL + filename, overview_url,
value, filter={"field": field, "value": name})
if not field["name"] in filtered_urls:
filtered_urls[field["name"]] = {}
filtered_urls[field["name"]][name] = os.path.basename(outputfile)
outputfile = os.path.join(outputdir,
common.filename_encode(server_type),
common.filename_encode(month),
common.filename_encode(filename),
"index.html")
generate_file_stats(outputfile, month, baseURL + filename, overview_url,
data, filtered_urls=filtered_urls)
if filename not in filedata:
filedata[filename] = {}
month_url = "%s/%s/%s" % (common.filename_encode(month),
common.filename_encode(filename),
"index.html")
filedata[filename][month] = {"url": month_url, "hits": data["hits"], "bandwidth": data["bandwidth"]}
monthdata = {}
for filename, data in filedata.iteritems():
outputfile = os.path.join(outputdir,
common.filename_encode(server_type),
"overview-" + common.filename_encode(filename + ".html"))
generate_file_overview(outputfile, baseURL + filename, data)
if current_month in data:
monthdata[filename] = dict(data[current_month])
outputfile = os.path.join(outputdir, common.filename_encode(server_type), "index.html")
generate_main_page(outputfile, current_month, baseURL, monthdata)
def generate_pages(datadir, outputdir):
for server_type, server_type_dir in get_names(datadir, True):
baseURL = get_config().get("stats", "baseURL_" + server_type)
filedata = {}
current_month = None
for month, month_dir in get_names(server_type_dir, True):
if current_month == None or month > current_month:
current_month = month
for filename, path in get_names(month_dir, False):
filename = re.sub(r"\.json$", "", filename)
with codecs.open(path, "rb", encoding="utf-8") as file:
data = json.load(file)
overview_url = "../../overview-" + common.filename_encode(
filename + ".html")
filtered_urls = {}
for field in common.fields:
if field["name"] not in data:
continue
# Create filtered views for the first thirty values of a field if they
# have filtered data.
sorted_field = get_template_environment(
).filters["sortfield"](data[field["name"]], field)
for name, value in sorted_field[0:get_default_count(field
)]:
if filter(lambda k: k not in ("hits", "bandwidth"),
value.iterkeys()):
outputfile = os.path.join(
outputdir, common.filename_encode(server_type),
common.filename_encode(month),
common.filename_encode(filename),
"filtered-%s-%s.html" % (
common.filename_encode(field["name"]),
common.filename_encode(name),
))
generate_file_stats(outputfile,
month,
baseURL + filename,
overview_url,
value,
filter={
"field": field,
"value": name
})
if not field["name"] in filtered_urls:
filtered_urls[field["name"]] = {}
filtered_urls[field["name"]][
name] = os.path.basename(outputfile)
outputfile = os.path.join(outputdir,
common.filename_encode(server_type),
common.filename_encode(month),
common.filename_encode(filename),
"index.html")
generate_file_stats(outputfile,
month,
baseURL + filename,
overview_url,
data,
filtered_urls=filtered_urls)
if filename not in filedata:
filedata[filename] = {}
month_url = "%s/%s/%s" % (common.filename_encode(month),
common.filename_encode(filename),
"index.html")
filedata[filename][month] = {
"url": month_url,
"hits": data["hits"],
"bandwidth": data["bandwidth"]
}
monthdata = {}
for filename, data in filedata.iteritems():
outputfile = os.path.join(
outputdir, common.filename_encode(server_type),
"overview-" + common.filename_encode(filename + ".html"))
generate_file_overview(outputfile, baseURL + filename, data)
if current_month in data:
monthdata[filename] = dict(data[current_month])
outputfile = os.path.join(outputdir,
common.filename_encode(server_type),
"index.html")
generate_main_page(outputfile, current_month, baseURL, monthdata)
