def compile_all_dimensions():
"""Get all dimensions in all stats, along with all their possible values"""
all_dims = {}
for file_ in traverse_files():
parsed = parser.parse(open(file_).read().decode(ENC), skip_data=True)
if 'outter_dim_key' not in parsed: continue
for dim, vals in parser.iter_dims(parsed):
try:
all_dims[dim].update(set(vals))
except KeyError:
all_dims[dim] = set(vals)
return all_dims
def filter_by_dims(dimensions):
"""Iterate through files and skip any that has dimensions not included in `dimensions`,
yields content"""
for file_ in traverse_files():
content = open(file_).read().decode(ENC)
parsed = parser.parse(content, skip_data=True)
# skip flat file
if 'outter_dim_values' not in parsed: continue
dims = set(dim for dim, _ in parser.iter_dims(parsed))
if len(dims) > len(dims.intersection(dimensions)):
# it has some dimensions not specified in arg, skip it
continue
yield content
def parse_and_save(content):
parsed = parser.parse(content)
cols = [TOP_DIMS[ dim ] for dim, _ in parser.iter_dims(parsed)]
conn = engine.connect()
trans = conn.begin()
conn.execute("insert into statinfo (statname) values (:statname) ",
statname=parsed['stat'])
print conn.execute("select currval(statinfo_id_seq) ").fetchall()[0][0]
return
for entry in parser.iter_data(parsed):
dim_vals = entry[:-1]
val = entry[-1]
for i, dim_val in enumerate(dim_vals):
print cols[i], dim_val
print val
break
