continue try: name = pw.format_string( row.get("nombre", row.get("comuna", pw.NO_DATA_UNICODE))) except: print(u"-Error: Can't read name for ID {0}, skipping.".format( idval)) continue try: fuel_string = row.get( "Tipo", row.get("Tipo ", row.get("Combustible", pw.NO_DATA_UNICODE))) fuel = pw.standardize_fuel(fuel_string, fuel_thesaurus) except: print(u"-Error: Can't read fuel for plant {0}.".format(name)) fuel = pw.NO_DATA_SET # special treament of name for biomass plants (not included in data file) if fuel == set(["Biomass"]): try: name = pw.format_string( plant_locations["Biomass"][idval][2]) except: print( "-Error: Can't read name for ID {0} (Biomass).".format( idval)) continue
plants_dictionary[idnr].capacity += unit_capacity # todo: average commissioning year calculation unit_month = int(rv[COLS_860_3['operating_month']]) unit_year_raw = int(rv[COLS_860_3['operating_year']]) unit_year = 1.0 * unit_year_raw + unit_month / 12 if idnr in commissioning_year_by_unit.keys(): commissioning_year_by_unit[idnr].append([unit_capacity, unit_year]) else: commissioning_year_by_unit[idnr] = [[unit_capacity, unit_year]] for i in COLS_860_3['fuel_type']: try: if rv[i] == "None": continue fuel_type = pw.standardize_fuel(rv[i], fuel_thesaurus) plants_dictionary[idnr].fuel.update(fuel_type) except: continue else: print("Can't find plant with ID: {0}".format(idnr)) # calculate and save average commissioning year for idnr, unit_vals in commissioning_year_by_unit.iteritems(): cap_times_year = 0 total_cap = 0 for unit in unit_vals: cap_times_year += unit[0] * unit[1] total_cap += unit[0] plants_dictionary[idnr].commissioning_year = cap_times_year / total_cap
idnr = pw.make_id(SAVE_CODE, int(rv[COLS_860_3['idnr']])) except: continue if idnr in plants_dictionary: unit_capacity = float(rv[COLS_860_3['capacity']]) plants_dictionary[idnr].capacity += unit_capacity unit_month = int(rv[COLS_860_3['operating_month']]) unit_year_raw = int(rv[COLS_860_3['operating_year']]) unit_year = 1.0 * unit_year_raw + unit_month / 12 if idnr in commissioning_year_by_unit: commissioning_year_by_unit[idnr].append([unit_capacity, unit_year]) else: commissioning_year_by_unit[idnr] = [ [unit_capacity, unit_year] ] primary_fuel = pw.standardize_fuel(rv[COLS_860_3['primary_fuel']], fuel_thesaurus, as_set=False) plants_dictionary[idnr].other_fuel.update(set([primary_fuel])) for i in COLS_860_3['other_fuel']: try: if rv[i] == "None": continue fuel_type = pw.standardize_fuel(rv[i], fuel_thesaurus, as_set=True) plants_dictionary[idnr].other_fuel.update(fuel_type) except: continue cap_fuel = plant_fuel_capacity[idnr].get(primary_fuel, 0) cap_fuel += unit_capacity plant_fuel_capacity[idnr][primary_fuel] = cap_fuel else: print("Can't find plant with ID: {0}".format(idnr))
continue # don't continue reading this line b/c it's not a full plant try: if rv[generation_col] == u'-': generation = pw.PlantGenerationObject() else: gen_gwh = float(rv[generation_col]) generation = pw.PlantGenerationObject.create(gen_gwh, DATA_YEAR, source=SOURCE_URL) except: print("-Error: Can't read generation for plant {0}".format(name)) generation = pw.PlantGenerationObject() try: plant_type = pw.format_string(rv[type_col]) if plant_type == u"HYDRO": fuel = pw.standardize_fuel(plant_type, fuel_thesaurus) elif plant_type == u"NUCLEAR": fuel = pw.standardize_fuel(plant_type, fuel_thesaurus) elif plant_type == u"THERMAL": fuel = pw.standardize_fuel(rv[fuel1_col], fuel_thesaurus) if rv[fuel2_col] and rv[fuel2_col] != 'n/a': fuel2 = pw.standardize_fuel(rv[fuel2_col], fuel_thesaurus) fuel = fuel.union(fuel2) else: print("Can't identify plant type {0}".format(plant_type)) except: print(u"Can't identify plant type for plant {0}".format(name)) # look up location if id_val in plant_locations: latitude = plant_locations[id_val]["latitude"]
try: name = pw.format_string( row.get("nombre", row.get("comuna", pw.NO_DATA_UNICODE))) except: print(u"-Error: Can't read name for ID {0}, skipping.".format( idval)) continue try: fuel_string = row.get( "Tipo", row.get("Tipo ", row.get("Combustible", pw.NO_DATA_UNICODE))) fuels = pw.standardize_fuel(fuel_string, fuel_thesaurus, as_set=True) except: print(u"-Error: Can't read fuel for plant {0}.".format(name)) primary_fuel = pw.NO_DATA_UNICODE other_fuels = pw.NO_DATA_SET.copy() else: if len(fuels) == 1: primary_fuel = fuels.pop() other_fuels = pw.NO_DATA_SET.copy() elif len(fuels) == 2: primary, secondary = fuel_string.split(' / ') primary_fuel = pw.standardize_fuel(primary, fuel_thesaurus, as_set=False) other_fuels = pw.standardize_fuel(secondary,
else: owner_string = previous_owner # get capacity try: capacity_value = float( rv[COLS['capacity']]) * CAPACITY_CONVERSION_TO_MW except: print("Cant read capacity for plant {0}.".format(name_string)) capacity_value = 0 # check if we've seen this plant before if name_string not in plant_names: # first time we've seen this plant fuel_type = pw.standardize_fuel(fuel_string, fuel_thesaurus, as_set=False) idnr = pw.format_string(aux_plant_info[name_string]['gppd_idnr']) new_plant = pw.PowerPlant(plant_idnr=idnr, plant_name=name_string, plant_owner=owner_string, plant_country=COUNTRY_NAME, plant_capacity=capacity_value, plant_primary_fuel=fuel_type, plant_cap_year=YEAR_OF_DATA, plant_source=SOURCE_NAME, plant_source_url=SOURCE_URL) plants_dictionary[idnr] = new_plant plant_names[name_string] = (idnr, {fuel_type: capacity_value}) # increment count count += 1
try: idnr = str(row[id_col]) if not idnr: # must have plant ID - don't read plant if not print(u"-Error: Null ID for plant {0}.".format(name)) continue except: print(u"-Error: Can't read ID for plant {0}.".format(name)) continue # must have plant ID - don't read plant if not try: capacity = float(pw.format_string(row[capacity_col].replace(",", ""))) # note: may need to convert to MW except: print(u"-Error: Can't read capacity for plant {0}; value: {1}".format(name, row[capacity_col])) capacity = pw.NO_DATA_NUMERIC # primary fuel try: primary_fuel = pw.standardize_fuel(row[primary_fuel_col], fuel_thesaurus, as_set=False) except: print(u"-Error: Can't read fuel type for plant {0}.".format(name)) primary_fuel = pw.NO_DATA_UNICODE # other fuels try: if row[other_fuel_col]: other_fuel = pw.standardize_fuel(row[other_fuel_col], fuel_thesaurus, as_set=True) else: other_fuel = pw.NO_DATA_SET.copy() except: print(u"-Error: Can't read secondary fuel type for plant {0}.".format(name)) other_fuel = pw.NO_DATA_SET.copy() try: latitude = float(row[latitude_col]) longitude = float(row[longitude_col])
try: idnr = int(row[id_col]) except: print(u"-Error: Can't read plant ID: {0}".format(row[id_col])) continue # must have ID number # skip non operational statuses if row[status_col] in NON_OPERATIONAL_STATUSES: continue try: capacity = float(row[capacity_col]) except: capacity = pw.NO_DATA_NUMERIC try: fuel = pw.standardize_fuel(row[fuel_col], fuel_thesaurus, as_set=False) except: print(u"-Error: Can't read fuel type for plant {0}.".format(name)) fuel = pw.NO_DATA_UNICODE try: latitude = float(row[latitude_col]) longitude = float(row[longitude_col]) geolocation_source = SAVE_CODE except: latitude, longitude = pw.NO_DATA_NUMERIC, pw.NO_DATA_NUMERIC geolocation_source = pw.NO_DATA_UNICODE try: owner = pw.format_string(row[owner_col]) except: print(u"-Error: Can't read owner for plant {0}.".format(name)) owner = pw.NO_DATA_UNICODE
try: capacity = float(capacity_val) except: try: capacity = eval(capacity_val) except: print("Could not evaluate {0}".format(capacity_val)) if year_built_val: year_built = int(year_built_val) if fuel_type_str: if '+' in fuel_type_str: fuel_type = None else: fuel_type = pw.standardize_fuel(fuel_type_str, fuel_thesaurus) if name_str: # if true, this row begins a new plant # first process the previous plant unless this is the first entry if i > START_ROW: print i print fuel_type_set raw_input() total_capacity = sum(capacity_list) average_year_built = sum(year_built_list) / len( year_built_list) # TODO: fix this new_location = pw.LocationObject(latitude=0.0, longitude=0.0) new_plant = pw.PowerPlant(plant_idnr=plant_idnr, plant_name=name, plant_country=COUNTRY_NAME, plant_capacity=total_capacity,
plant_oid = plant.find("Electricity_Infrastructure:OBJECTID", ns).text # check if plant is already known, and skip if there is not a record (includes cases where AREMI has duplicated plants) if plant_oid not in linking_table: print(u"Error: Don't have prescribed ID for plant {0}; OID={1}.". format(name, plant_oid)) continue # get the assigned GPPD IDNR as an int, stripping the 'AUS' prefix plant_id = int(linking_table[plant_oid]['gppd_idnr'][3:]) try: owner = pw.format_string( plant.find("Electricity_Infrastructure:OWNER", ns).text) except: owner = pw.NO_DATA_UNICODE primary_fuel = pw.standardize_fuel( plant.find("Electricity_Infrastructure:PRIMARYFUELTYPE", ns).text, fuel_thesaurus) try: capacity = plant.find("Electricity_Infrastructure:GENERATIONMW", ns).text capacity = float(capacity) except: print(u"Error: Can't read capacity for plant {0}.".format(name)) capacity = pw.NO_DATA_NUMERIC coords = plant.find( "Electricity_Infrastructure:SHAPE/gml:Point/gml:pos", ns).text.split(" ") try: longitude = float(coords[0]) latitude = float(coords[1]) geolocation_source = SOURCE_NAME
print(u"- Error: Can't get name for plant {0}".format(id_val)) continue # get capacity in MW try: capacity = float(p_dict['potenciaInstalada'].strip(" MW")) except: print(u"- Error: Can't get capacity for plant {0}".format(id_val)) continue # get fuel type try: fuel_string_raw = p_dict['icon'] fuel_type = parse_fuel_URY(fuel_string_raw[9:12], id_val) # extract fuel name from icon name fuel = pw.standardize_fuel(fuel_type, fuel_thesaurus) except: print(u"- Error: Can't read fuel type for plant {0}".format(id_val)) continue # get coordinates try: c_dict = p_dict['latlng'] latitude = float(p_dict['latlng']['lat']) longitude = float(p_dict['latlng']['lng']) geolocation_source = SOURCE_NAME except: print(u"- Error: Can't read coordinates for plant {0}".format(id_val)) geolocation_source = pw.NO_DATA_UNICODE # get owner
rv = sheet.row_values(i) try: ref = int(rv[ref_col]) if not ref: print("-Error reading ref from: {0}".format(rv[ref_col])) continue except: continue try: project_type = pw.format_string(rv[type_col]) if project_type not in PROJECT_TYPES_TO_READ: # don't read all project types continue try: fuel = pw.standardize_fuel(project_type, fuel_thesaurus) except: print("-Error reading fuel: {0}".format(project_type)) fuel = pw.NO_DATA_SET except: print(u"-Error: Can't read project type for project {0}.".format(ref)) continue try: status = pw.format_string(rv[status_col]) if status != u"Registered": continue except: print( u"-Error: Can't read project status for project {0}.".format(ref)) continue
# TODO: organize this into fiscal year (april through march) generation_13 = get_CEA_generation(rv, gen_13_14_col, 2013, SOURCE_NAME) generation_14 = get_CEA_generation(rv, gen_14_15_col, 2014, SOURCE_NAME) generation_15 = get_CEA_generation(rv, gen_15_16_col, 2015, SOURCE_NAME) generation_16 = get_CEA_generation(rv, gen_16_17_col, 2016, SOURCE_NAME) generation_17 = get_CEA_generation(rv, gen_17_18_col, 2017, SOURCE_NAME) generation = [ generation_13, generation_14, generation_15, generation_16, generation_17 ] try: plant_type = pw.format_string(rv[type_col]) if plant_type in [u"HYDRO", u"NUCLEAR"]: primary_fuel = pw.standardize_fuel(plant_type, fuel_thesaurus, as_set=False) other_fuel = pw.NO_DATA_SET.copy() elif plant_type == u"THERMAL": primary_fuel = pw.standardize_fuel(rv[primary_fuel_col], fuel_thesaurus, as_set=False) if rv[other_fuel_col] and rv[other_fuel_col] != 'n/a': other_fuel = pw.standardize_fuel(rv[other_fuel_col], fuel_thesaurus, as_set=True) else: other_fuel = pw.NO_DATA_SET.copy() else: print("Can't identify plant type {0}".format(plant_type)) except:
datareader = csv.reader(f) headers = datareader.next() for row in datareader: commissioning_years_dictionary[pw.format_string(row[0])] = row[1] # read data from csv and parse count = 1 wb = xlrd.open_workbook(RAW_FILE_NAME) ws = wb.sheet_by_name(TAB) # treat first data row specially for plant name rv0 = ws.row_values(START_ROW) current_plant_name = pw.format_string(rv0[COLS['name']]) current_owner = pw.format_string(rv0[COLS['owner']]) current_fuel_types = pw.standardize_fuel(rv0[COLS['fuel']], fuel_thesaurus) current_capacity_sum = float(rv0[COLS['capacity']]) * CAPACITY_CONVERSION_TO_MW current_generation_sum = float( rv0[COLS['generation']]) * GENERATION_CONVERSION_TO_GWH test_print = False for row_id in range(START_ROW + 1, ws.nrows): rv = ws.row_values(row_id) row_fuel = pw.format_string(rv[COLS['fuel']], None) row_name = pw.format_string(rv[COLS['name']], None) row_grid = pw.format_string(rv[COLS['grid']], None) if row_grid == u"AISLADO": continue # don't add islanded generators (not grid-connected)
# read each row in the file count = 1 for row in datareader: try: name = pw.format_string(row[name_col]) except: print(u"Error: Can't read plant name.") continue # must have plant name - don't read plant if not try: capacity = float(pw.format_string( row[capacity_col])) # note: may need to convert to MW except: print(u"Error: Can't read capacity for plant {0}.".format(name)) try: fuel = pw.standardize_fuel(row[fuel_col], fuel_thesaurus) except: print(u"Error: Can't read fuel type for plant {0}.".format(name)) try: latitude = float(row[latitude_col]) longitude = float(row[longitude_col]) except: latitude, longitude = pw.NO_DATA_NUMERIC, pw.NO_DATA_NUMERIC # assign ID number idnr = pw.make_id(SAVE_CODE, count) new_location = pw.LocationObject(pw.NO_DATA_UNICODE, latitude, longitude) new_plant = pw.PowerPlant(plant_idnr=idnr, plant_name=name, plant_country=country,
continue # get capacity in MW try: capacity = float(p_dict['potenciaInstalada'].strip(" MW")) except: print(u"- Error: Can't get capacity for plant {0}".format(id_val)) continue # get fuel type try: fuel_string_raw = p_dict['icon'] primary_fuel_string, other_fuel_set = parse_fuel_URY( fuel_string_raw[9:12], id_val) # extract fuel name from icon name primary_fuel = pw.standardize_fuel(primary_fuel_string, fuel_thesaurus, as_set=False) except: print(u"- Error: Can't read fuel type {0} for plant {1}".format( primary_fuel_string, id_val)) continue # get coordinates try: c_dict = p_dict['latlng'] latitude = float(p_dict['latlng']['lat']) longitude = float(p_dict['latlng']['lng']) geolocation_source = SOURCE_NAME except: print(u"- Error: Can't read coordinates for plant {0}".format(id_val)) geolocation_source = pw.NO_DATA_UNICODE
except: print(u"-Error: Can't read plant name.") continue # must have plant name - don't read plant if not try: idnr = int(row[id_col]) except: print(u"-Error: Can't read ref id.") continue # must have ID number try: capacity = float(pw.format_string( row[capacity_col])) # note: may need to convert to MW except: print(u"-Error: Can't read capacity for plant {0}.".format(name)) capacity = 0.0 try: fuel_type = pw.standardize_fuel(row[fuel_col], fuel_thesaurus) if not fuel_type: print("-Error: No fuel type for {0}.".format(row[fuel_col])) except: print("-Error: Can't read fuel for plant {0}.".format(name)) fuel_type = set([]) try: x_coordinate = float(row[x_coordinate_col].replace(",", "")) y_coordinate = float(row[y_coordinate_col].replace(",", "")) longitude, latitude = pyproj.transform(osgb36, wgs84, x_coordinate, y_coordinate) geolocation_source = SOURCE_NAME_REPD except: print(u"-Error: Can't read location for plant {0}.".format(name)) latitude, longitude = pw.NO_DATA_NUMERIC, pw.NO_DATA_NUMERIC geolocation_source = pw.NO_DATA_UNICODE
print(u"-Error: No name on row {0}".format(i + 1)) continue except: print(u"-Error: Can't read name of plant on row {0}".format(i + 1)) name = pw.NO_DATA_UNICODE # without this, next pass thru loop uses old name continue try: owner = pw.format_string(row[owner_col], None) except: print(u"-Error: Can't read owner of plant with name {0}".format(name)) owner = pw.NO_DATA_UNICODE try: primary_fuel = pw.standardize_fuel(row[fuel_col], fuel_thesaurus, as_set=False) except: print(u"-Error: Can't read fuel for plant with name {0}".format(name)) primary_fuel = pw.NO_DATA_UNICODE try: capacity = float(row[capacity_col]) except: print(u"-Error: Can't read capacity for plant with name {0}".format( name)) capacity = pw.NO_DATA_NUMERIC try: latitude = float(row[latitude_col]) longitude = float(row[longitude_col])