Example #1
0
def compile_all_dimensions():
    """Get all dimensions in all stats, along with all their possible values"""
    all_dims = {}
    for file_ in traverse_files():
        parsed = parser.parse(open(file_).read().decode(ENC), skip_data=True)
        if 'outter_dim_key' not in parsed: continue
        for dim, vals in parser.iter_dims(parsed):
            try:
                all_dims[dim].update(set(vals))
            except KeyError:
                all_dims[dim] = set(vals)

    return all_dims
Example #2
0
def filter_by_dims(dimensions):
    """Iterate through files and skip any that has dimensions not included in `dimensions`,
       yields content"""
    for file_ in traverse_files():
        content = open(file_).read().decode(ENC)
        parsed = parser.parse(content, skip_data=True)

        # skip flat file
        if 'outter_dim_values' not in parsed: continue

        dims = set(dim for dim, _ in parser.iter_dims(parsed))
        if len(dims) > len(dims.intersection(dimensions)):
            # it has some dimensions not specified in arg, skip it
            continue

        yield content
Example #3
0
def parse_and_save(content):
    parsed = parser.parse(content)
    cols = [TOP_DIMS[ dim ] for dim, _ in parser.iter_dims(parsed)]

    conn = engine.connect()
    trans = conn.begin()

    conn.execute("insert into statinfo (statname) values (:statname) ",
                 statname=parsed['stat'])
    print conn.execute("select currval(statinfo_id_seq) ").fetchall()[0][0]
    return
    for entry in parser.iter_data(parsed):
        dim_vals = entry[:-1]
        val = entry[-1]

        for i, dim_val in enumerate(dim_vals):
            print cols[i], dim_val

        print val


        break