def test_fileencoding(self): tests = [ ("foo_bar", True), ("1234.txt", True), ("xYz.DAT", True), ("foo/bar.txt", False), ("foo/bar+bas-xyz.txt", False), (u"foo\u1234-bar\u4321", False), ("foobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobar", u"foobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobarfoobar\u2026" ) ] for name, expect_identical in tests: path = common.filename_encode(name) if expect_identical == True: self.assertEqual( path, name, "Encoding '%s' shouldn't change string" % name) else: self.assertTrue( re.match(r"^[\w\.\-]*$", path), "Encoding '%s' should replace all special characters" % name) if isinstance(expect_identical, basestring): self.assertEqual( common.filename_decode(path), expect_identical, "Encoding and decoding '%s' should produce a truncated string as result" % name) else: self.assertEqual( common.filename_decode(path), name, "Encoding and decoding '%s' should produce the original string" % name)
def save_stats(server_type, data, factor=1): base_dir = os.path.join(get_config().get("stats", "dataDirectory"), common.filename_encode(server_type)) for month, month_data in data.iteritems(): for name, file_data in month_data.iteritems(): path = os.path.join(base_dir, common.filename_encode(month), common.filename_encode(name + ".json")) if os.path.exists(path): with codecs.open(path, "rb", encoding="utf-8") as fileobj: existing = json.load(fileobj) else: existing = {} merge_objects(existing, file_data, factor) dir = os.path.dirname(path) try: os.makedirs(dir) except OSError, e: if e.errno != errno.EEXIST: raise with codecs.open(path, "wb", encoding="utf-8") as fileobj: json.dump(existing, fileobj, indent=2, sort_keys=True)
def test_fileencoding(self): tests = [ ("foo_bar", True), ("1234.txt", True), ("xYz.DAT", True), ("foo/bar.txt", False), ("foo/bar+bas-xyz.txt", False), (u"foo\u1234-bar\u4321", False), ] for name, expect_identical in tests: path = common.filename_encode(name) if expect_identical: self.assertEqual(path, name, "Encoding '%s' shouldn't change string" % name) else: self.assertTrue(re.match(r"^[\w\.\-]*$", path), "Encoding '%s' should replace all special characters" % name) self.assertEqual(common.filename_decode(path), name, "Encoding and decoding '%s' should produce the original string" % name)
def generate_pages(datadir, outputdir): for server_type, server_type_dir in get_names(datadir, True): baseURL = get_config().get('stats', 'baseURL_' + server_type) filedata = {} current_month = None for month, month_dir in get_names(server_type_dir, True): if current_month == None or month > current_month: current_month = month for filename, path in get_names(month_dir, False): filename = re.sub(r'\.json$', '', filename) with codecs.open(path, 'rb', encoding='utf-8') as file: data = json.load(file) overview_url = '../../overview-' + common.filename_encode(filename + '.html') filtered_urls = {} for field in common.fields: if field['name'] not in data: continue # Create filtered views for the first thirty values of a field if they # have filtered data. sorted_field = get_template_environment().filters['sortfield'](data[field['name']], field) for name, value in sorted_field[0:get_default_count(field)]: if filter(lambda k: k not in ('hits', 'bandwidth'), value.iterkeys()): outputfile = os.path.join(outputdir, common.filename_encode(server_type), common.filename_encode(month), common.filename_encode(filename), 'filtered-%s-%s.html' % ( common.filename_encode(field['name']), common.filename_encode(name), )) generate_file_stats(outputfile, month, baseURL + filename, overview_url, value, filter={'field': field, 'value': name}) if not field['name'] in filtered_urls: filtered_urls[field['name']] = {} filtered_urls[field['name']][name] = os.path.basename(outputfile) outputfile = os.path.join(outputdir, common.filename_encode(server_type), common.filename_encode(month), common.filename_encode(filename), 'index.html') generate_file_stats(outputfile, month, baseURL + filename, overview_url, data, filtered_urls=filtered_urls) if filename not in filedata: filedata[filename] = {} month_url = '%s/%s/%s' % (common.filename_encode(month), common.filename_encode(filename), 'index.html') filedata[filename][month] = {'url': month_url, 'hits': data['hits'], 'bandwidth': data['bandwidth']} monthdata = {} for filename, data in filedata.iteritems(): outputfile = os.path.join(outputdir, common.filename_encode(server_type), 'overview-' + common.filename_encode(filename + '.html')) generate_file_overview(outputfile, baseURL + filename, data) if current_month in data: monthdata[filename] = dict(data[current_month]) outputfile = os.path.join(outputdir, common.filename_encode(server_type), 'index.html') generate_main_page(outputfile, current_month, baseURL, monthdata)
def generate_pages(datadir, outputdir): for server_type, server_type_dir in get_names(datadir, True): baseURL = get_config().get("stats", "baseURL_" + server_type) filedata = {} current_month = None for month, month_dir in get_names(server_type_dir, True): if current_month == None or month > current_month: current_month = month for filename, path in get_names(month_dir, False): filename = re.sub(r"\.json$", "", filename) with codecs.open(path, "rb", encoding="utf-8") as file: data = simplejson.load(file) overview_url = "../../overview-" + common.filename_encode(filename + ".html") filtered_urls = {} for field in common.fields: if field["name"] not in data: continue # Create filtered views for the first thirty values of a field if they # have filtered data. for name, value in get_template_environment().filters["sortfield"](data[field["name"]], field)[0:30]: if filter(lambda k: k not in ("hits", "bandwidth"), value.iterkeys()): outputfile = os.path.join(outputdir, common.filename_encode(server_type), common.filename_encode(month), common.filename_encode(filename), "filtered-%s-%s.html" % ( common.filename_encode(field["name"]), common.filename_encode(name), )) generate_file_stats(outputfile, month, baseURL + filename, overview_url, value, filter={"field": field, "value": name}) if not field["name"] in filtered_urls: filtered_urls[field["name"]] = {} filtered_urls[field["name"]][name] = os.path.basename(outputfile) outputfile = os.path.join(outputdir, common.filename_encode(server_type), common.filename_encode(month), common.filename_encode(filename), "index.html") generate_file_stats(outputfile, month, baseURL + filename, overview_url, data, filtered_urls=filtered_urls) if filename not in filedata: filedata[filename] = {} month_url = "%s/%s/%s" % (common.filename_encode(month), common.filename_encode(filename), "index.html") filedata[filename][month] = {"url": month_url, "hits": data["hits"], "bandwidth": data["bandwidth"]} monthdata = {} for filename, data in filedata.iteritems(): outputfile = os.path.join(outputdir, common.filename_encode(server_type), "overview-" + common.filename_encode(filename + ".html")) generate_file_overview(outputfile, baseURL + filename, data) if current_month in data: monthdata[filename] = dict(data[current_month]) outputfile = os.path.join(outputdir, common.filename_encode(server_type), "index.html") generate_main_page(outputfile, current_month, baseURL, monthdata)
def generate_pages(datadir, outputdir): for server_type, server_type_dir in get_names(datadir, True): baseURL = get_config().get("stats", "baseURL_" + server_type) filedata = {} current_month = None for month, month_dir in get_names(server_type_dir, True): if current_month == None or month > current_month: current_month = month for filename, path in get_names(month_dir, False): filename = re.sub(r"\.json$", "", filename) with codecs.open(path, "rb", encoding="utf-8") as file: data = json.load(file) overview_url = "../../overview-" + common.filename_encode( filename + ".html") filtered_urls = {} for field in common.fields: if field["name"] not in data: continue # Create filtered views for the first thirty values of a field if they # have filtered data. sorted_field = get_template_environment( ).filters["sortfield"](data[field["name"]], field) for name, value in sorted_field[0:get_default_count(field )]: if filter(lambda k: k not in ("hits", "bandwidth"), value.iterkeys()): outputfile = os.path.join( outputdir, common.filename_encode(server_type), common.filename_encode(month), common.filename_encode(filename), "filtered-%s-%s.html" % ( common.filename_encode(field["name"]), common.filename_encode(name), )) generate_file_stats(outputfile, month, baseURL + filename, overview_url, value, filter={ "field": field, "value": name }) if not field["name"] in filtered_urls: filtered_urls[field["name"]] = {} filtered_urls[field["name"]][ name] = os.path.basename(outputfile) outputfile = os.path.join(outputdir, common.filename_encode(server_type), common.filename_encode(month), common.filename_encode(filename), "index.html") generate_file_stats(outputfile, month, baseURL + filename, overview_url, data, filtered_urls=filtered_urls) if filename not in filedata: filedata[filename] = {} month_url = "%s/%s/%s" % (common.filename_encode(month), common.filename_encode(filename), "index.html") filedata[filename][month] = { "url": month_url, "hits": data["hits"], "bandwidth": data["bandwidth"] } monthdata = {} for filename, data in filedata.iteritems(): outputfile = os.path.join( outputdir, common.filename_encode(server_type), "overview-" + common.filename_encode(filename + ".html")) generate_file_overview(outputfile, baseURL + filename, data) if current_month in data: monthdata[filename] = dict(data[current_month]) outputfile = os.path.join(outputdir, common.filename_encode(server_type), "index.html") generate_main_page(outputfile, current_month, baseURL, monthdata)