Esempio n. 1
0
File: parser.py Progetto: sonya/eea
def parse_io():
    # choose 中分類 for all io tables.
    # 中分類 for 1990 and 1995 don't break down the electronic
    # sectors as far as i would like, so use 小分類
    files = {
        1990: "l00_21.xls",
        1995: "l00_21.xls",
        2000: "io00a301.xls",
        2005: "io05a301.xls",
    }

    tables = HybridTableCreator(config.SCHEMA)

    for (year, filename) in files.items():
        # 1995 and 2000 io tables: easiest
        tables.add_io_table(year)
        codes = tables.new_sector_codes(year)

        # for 1995 use the heisei 2-7-12 file since it has more
        # harmonized sectors than the standalone 1995 file
        if year == 1995:
            sheetindex = 2
        else:
            # the first page of the heisei 2-7-12 file (used for 1990)
            # happens to be 1990 at nominal prices, matching the others
            sheetindex = 0

        path = fileutils.getcache(filename, "jp", str(year))
        wb = xlrd.open_workbook(path)
        sheet = wb.sheet_by_index(sheetindex)
        ind_names = None
        ind_codes = None
        for i in range(sheet.nrows):
            row = sheet.row_values(i)
            if ind_codes is None:
                for cell in row:
                    if cell == 1:
                        ind_codes = [str(c).strip().rjust(3, "0") for c in row]
                        break
                    if cell.strip() == "001":
                        ind_codes = row
                        break
            elif ind_names is None:
                ind_names = row
                temp_codes = [None, None]
                for i in range(2, len(row)):
                    temp_codes.append(codes.set_code(ind_codes[i], row[i]))
                ind_codes = temp_codes
            else:
                from_code = row[0]
                if type(from_code) is float:
                    from_code = str(int(from_code)).rjust(3, "0")
                from_code = codes.set_code(from_code, row[1])
                if from_code:
                    for i in range(2, len(row)):
                        to_code = ind_codes[i]
                        value = row[i]
                        tables.insert_io(year, from_code, to_code, value)

        codes.update_codes()
Esempio n. 2
0
File: parser.py Progetto: sonya/eea
def parse_io():
    files = {
        2005: fileutils.getdatapath("2005年42部门投入产出流量表.xls", "cn-io"),
        2007: fileutils.getdatapath(
            "0101.xls", "cn-io", "中国投入产出表2007", "excel"),
        }

    tables = HybridTableCreator(config.SCHEMA)

    for (year, filename) in files.items():
        tables.add_io_table(year)
        codes = tables.new_sector_codes(year)

        wb = xlrd.open_workbook(filename)
        # in 2005 sheet 0 is x10k RMB, 2007 has only 1 sheet @x10k RMB
        sheet = wb.sheet_by_index(0)

        ind_codes = None

        # the excel files also have this evil problem of merging
        # cells for appearance and not meaning.  we only have 2
        # years so curate them
        codes.set_code("FU101", "农村居民消费")
        codes.set_code("FU102", "城镇居民消费")
        codes.set_code("FU103", "政府消费支出")
        codes.set_code("FU201", "固定资本形成总额")
        codes.set_code("FU202", "存货增加")
        codes.set_code("GCF", "资本形成合计")
        codes.set_code("EX", "出口")

        codes.blacklist_code("TI")
        codes.blacklist_code("TII")

        for i in range(sheet.nrows):
            row = sheet.row_values(i)
            if ind_codes is None:
                for cell in row:
                    if type(cell) is str and cell.strip("0") == "1":
                        ind_codes = []
                        break
                if ind_codes is not None:
                    for cell in row[3:]:
                        if type(cell) is float:
                            cell = str(int(cell))
                        if regexes.is_num(cell) or table.has_code(cell):
                            ind_codes.append(cell)
                        else:
                            ind_codes.append(None)
            else:
                from_code = codes.set_code(row[2], row[1])
                if from_code:
                    for (value, to_code) in zip(row[3:], ind_codes):
                        if to_code is not None:
                            tables.insert_io(year, from_code, to_code, value)

        codes.update_codes()
Esempio n. 3
0
File: parser.py Progetto: sonya/eea
def parse_io():
    files = {
        2005: fileutils.getdatapath("2005年42部门投入产出流量表.xls", "cn-io"),
        2007: fileutils.getdatapath("0101.xls", "cn-io", "中国投入产出表2007",
                                    "excel"),
    }

    tables = HybridTableCreator(config.SCHEMA)

    for (year, filename) in files.items():
        tables.add_io_table(year)
        codes = tables.new_sector_codes(year)

        wb = xlrd.open_workbook(filename)
        # in 2005 sheet 0 is x10k RMB, 2007 has only 1 sheet @x10k RMB
        sheet = wb.sheet_by_index(0)

        ind_codes = None

        # the excel files also have this evil problem of merging
        # cells for appearance and not meaning.  we only have 2
        # years so curate them
        codes.set_code("FU101", "农村居民消费")
        codes.set_code("FU102", "城镇居民消费")
        codes.set_code("FU103", "政府消费支出")
        codes.set_code("FU201", "固定资本形成总额")
        codes.set_code("FU202", "存货增加")
        codes.set_code("GCF", "资本形成合计")
        codes.set_code("EX", "出口")

        codes.blacklist_code("TI")
        codes.blacklist_code("TII")

        for i in range(sheet.nrows):
            row = sheet.row_values(i)
            if ind_codes is None:
                for cell in row:
                    if type(cell) is str and cell.strip("0") == "1":
                        ind_codes = []
                        break
                if ind_codes is not None:
                    for cell in row[3:]:
                        if type(cell) is float:
                            cell = str(int(cell))
                        if regexes.is_num(cell) or table.has_code(cell):
                            ind_codes.append(cell)
                        else:
                            ind_codes.append(None)
            else:
                from_code = codes.set_code(row[2], row[1])
                if from_code:
                    for (value, to_code) in zip(row[3:], ind_codes):
                        if to_code is not None:
                            tables.insert_io(year, from_code, to_code, value)

        codes.update_codes()
Esempio n. 4
0
File: parser.py Progetto: sonya/eea
def parse_io():
    # choose 中分類 for all io tables.
    # 中分類 for 1990 and 1995 don't break down the electronic
    # sectors as far as i would like, so use 小分類
    files = {
        1990: "l00_21.xls",
        1995: "l00_21.xls",
        2000: "io00a301.xls",
        2005: "io05a301.xls",
        }

    tables = HybridTableCreator(config.SCHEMA)

    for (year, filename) in files.items():
        # 1995 and 2000 io tables: easiest
        tables.add_io_table(year)
        codes = tables.new_sector_codes(year)

        # for 1995 use the heisei 2-7-12 file since it has more
        # harmonized sectors than the standalone 1995 file
        if year == 1995:
            sheetindex = 2
        else:
            # the first page of the heisei 2-7-12 file (used for 1990)
            # happens to be 1990 at nominal prices, matching the others
            sheetindex = 0

        path = fileutils.getcache(filename, "jp", str(year))
        wb = xlrd.open_workbook(path)
        sheet = wb.sheet_by_index(sheetindex)
        ind_names = None
        ind_codes = None
        for i in range(sheet.nrows):
            row = sheet.row_values(i)
            if ind_codes is None:
                for cell in row:
                    if cell == 1:
                        ind_codes = [str(c).strip().rjust(3, "0")
                                      for c in row]
                        break
                    if cell.strip() == "001":
                        ind_codes = row
                        break
            elif ind_names is None:
                ind_names = row
                temp_codes = [None, None]
                for i in range(2, len(row)):
                    temp_codes.append(
                        codes.set_code(ind_codes[i], row[i]))
                ind_codes = temp_codes
            else:
                from_code = row[0]
                if type(from_code) is float:
                    from_code = str(int(from_code)).rjust(3, "0")
                from_code = codes.set_code(from_code, row[1])
                if from_code:
                    for i in range(2, len(row)):
                        to_code = ind_codes[i]
                        value = row[i]
                        tables.insert_io(year, from_code, to_code, value)
 
        codes.update_codes()