plants_dictionary = {}
# load GEO and CARMA for matching coordinates
geo_database = pw.load_database(GEO_DATABASE_FILE)
print("Loaded {0} plants from GEO database.".format(len(geo_database)))
carma_database = pw.load_database(CARMA_DATABASE_FILE)
print("Loaded {0} plants from CARMA database.".format(len(carma_database)))
# read in plant matches file
with open(PLANT_MATCHES, "rbU") as f:
f.readline() # skip headers
csvreader = csv.reader(f)
plant_matches = {}
for row in csvreader:
dukes_name = str(row[0])
geo_id = pw.make_id(SAVE_CODE_GEO, int(row[1])) if row[1] else ""
carma_id = pw.make_id(SAVE_CODE_CARMA, int(row[2])) if row[2] else ""
repd_id = int(row[3]) if row[3] else ""
plant_matches[dukes_name] = {
"geo_id": geo_id,
"carma_id": carma_id,
"repd_id": repd_id
}
# load and process Renewable Energy Planning Database
print(u"Reading in plants...")
country = COUNTRY_NAME
with open(RAW_FILE_NAME_REPD, "rU") as f:
datareader = csv.reader(f)
headers = datareader.next()
while "Ref ID" not in headers: # find header row
name))
capacity = pw.NO_DATA_NUMERIC
try:
owner_string = row.get(
"Propietario", row.get("Propiedad", pw.NO_DATA_UNICODE))
if not owner_string:
print("-Error: No owner string for plant {0}".format(name))
owner = pw.NO_DATA_UNICODE
owner = pw.format_string(owner_string)
except:
print(u"-Error: Can't read owner for plant {0}.".format(name))
owner = pw.NO_DATA_UNICODE
# assign ID number, make PowerPlant object, add to dictionary
idnr = pw.make_id(SAVE_CODE, dataset["idstart"] + idval)
new_location = pw.LocationObject(pw.NO_DATA_UNICODE, latitude,
longitude)
new_plant = pw.PowerPlant(plant_idnr=idnr,
plant_name=name,
plant_country=COUNTRY_NAME,
plant_owner=owner,
plant_location=new_location,
plant_coord_source=geolocation_source,
plant_fuel=fuel,
plant_capacity=capacity,
plant_cap_year=YEAR_POSTED,
plant_source=SOURCE_NAME,
plant_source_url=SOURCE_URL)
plants_dictionary[idnr] = new_plant
capacity = float(pw.format_string(
row[capacity_col])) # note: may need to convert to MW
except:
print(u"Error: Can't read capacity for plant {0}.".format(name))
try:
fuel = pw.standardize_fuel(row[fuel_col], fuel_thesaurus)
except:
print(u"Error: Can't read fuel type for plant {0}.".format(name))
try:
latitude = float(row[latitude_col])
longitude = float(row[longitude_col])
except:
latitude, longitude = pw.NO_DATA_NUMERIC, pw.NO_DATA_NUMERIC
# assign ID number
idnr = pw.make_id(SAVE_CODE, count)
new_location = pw.LocationObject(pw.NO_DATA_UNICODE, latitude,
longitude)
new_plant = pw.PowerPlant(plant_idnr=idnr,
plant_name=name,
plant_country=country,
plant_location=new_location,
plant_fuel=fuel,
plant_capacity=capacity,
plant_source=SOURCE_NAME,
plant_source_url=SOURCE_URL)
plants_dictionary[idnr] = new_plant
count += 1
# report on plants read from file
print(u"Loaded {0} plants to database.".format(len(plants_dictionary)))
ws923_2013 = wb923_2013.sheet_by_name(TAB_NAME_923_2_2013)
print("Loading Form 860-2")
wb860_2 = xlrd.open_workbook(RAW_FILE_NAME_860_2)
ws860_2 = wb860_2.sheet_by_name(TAB_NAME_860_2)
print("Loading Form 860-3")
wb860_3 = xlrd.open_workbook(RAW_FILE_NAME_860_3)
ws860_3 = wb860_3.sheet_by_name(TAB_NAME_860_3)
# read in plants from File 2 of EIA-860
print("Reading in plants...")
plants_dictionary = {}
for row_id in xrange(2, ws860_2.nrows):
rv = ws860_2.row_values(row_id) # row value
name = pw.format_string(rv[COLS_860_2['name']])
idnr = pw.make_id(SAVE_CODE, int(rv[COLS_860_2['idnr']]))
capacity = 0.0
generation = pw.PlantGenerationObject()
owner = pw.format_string(str(rv[COLS_860_2['owner']]))
try:
latitude = float(rv[COLS_860_2['lat']])
except:
latitude = pw.NO_DATA_NUMERIC
try:
longitude = float(rv[COLS_860_2['lng']])
except:
longitude = pw.NO_DATA_NUMERIC
location = pw.LocationObject(u"", latitude, longitude)
new_plant = pw.PowerPlant(idnr, name, plant_country=COUNTRY_NAME,
plant_location=location, plant_coord_source=SOURCE_NAME,
plant_owner=owner, plant_capacity=capacity,
print(u"Error: Can't read owner for plant {0}.".format(name))
owner = u"Unknown"
try:
country = pw.standardize_country(row[country_col],
country_thesaurus)
except:
print("Error: Can't read country for plant {0}.".format(name))
country = u"Unknown"
# try:
# generation_GWh = float(row[generation_col]) / 1000
# except:
# print("Error: Can't read generation for plant {0}".format(name))
# generation_GWh = 0.0
# assign ID number
idnr = pw.make_id(SAVE_CODE, idnr)
new_location = pw.LocationObject("", latitude, longitude)
new_plant = pw.PowerPlant(plant_idnr=idnr,
plant_name=name,
plant_country=country,
plant_location=new_location,
plant_source=SOURCE_NAME,
plant_source_url=SOURCE_URL,
plant_owner=owner)
plants_dictionary[idnr] = new_plant
count += 1
# report on plants read from file
print(u"...read {0} plants.".format(len(plants_dictionary)))
# write database to csv format
except:
print(u"Can't identify plant type for plant {0}".format(name))
# look up location
if id_val in plant_locations:
latitude = plant_locations[id_val]["latitude"]
longitude = plant_locations[id_val]["longitude"]
geolocation_source = SOURCE_NAME
else:
print("-Error: Can't find CEA ID {0} in plant location file.".format(id_val))
latitude = pw.NO_DATA_NUMERIC
longitude = pw.NO_DATA_NUMERIC
geolocation_source = pw.NO_DATA_UNICODE
# assign ID number
idnr = pw.make_id(SAVE_CODE, id_val)
new_location = pw.LocationObject(pw.NO_DATA_UNICODE, latitude, longitude)
new_plant = pw.PowerPlant(plant_idnr=idnr, plant_name=name, plant_country=COUNTRY_NAME,
plant_location=new_location, plant_coord_source=geolocation_source,
plant_fuel=fuel,
plant_capacity=capacity, plant_cap_year=DATA_YEAR,
plant_source=SOURCE_NAME, plant_source_url=SOURCE_URL,
plant_generation=generation)
plants_dictionary[idnr] = new_plant
# now find average commissioning year weighted by capacity
for id_val, units in unit_list.iteritems():
# get plant from dictionary
plant_id = pw.make_id(SAVE_CODE, id_val)
plant = plants_dictionary[plant_id]
wb1 = xlrd.open_workbook(RAW_FILE_NAME_923_2)
ws1 = wb1.sheet_by_name(TAB_NAME_923_2)
print("Loading Form 860-2")
wb2 = xlrd.open_workbook(RAW_FILE_NAME_860_2)
ws2 = wb2.sheet_by_name(TAB_NAME_860_2)
print("Loading Form 860-3")
wb3 = xlrd.open_workbook(RAW_FILE_NAME_860_3)
ws3 = wb3.sheet_by_name(TAB_NAME_860_3)
# read in plants from File 2 of EIA-860
print("Reading in plants...")
plants_dictionary = {}
for row_id in xrange(2, ws2.nrows):
rv = ws2.row_values(row_id) # row value
name = pw.format_string(rv[COLS_860_2['name']])
idnr = pw.make_id(SAVE_CODE, int(rv[COLS_860_2['idnr']]))
capacity = 0.0
generation = pw.PlantGenerationObject()
owner = pw.format_string(str(rv[COLS_860_2['owner']]))
try:
latitude = float(rv[COLS_860_2['lat']])
except:
latitude = pw.NO_DATA_NUMERIC
try:
longitude = float(rv[COLS_860_2['lng']])
except:
longitude = pw.NO_DATA_NUMERIC
location = pw.LocationObject(u"", latitude, longitude)
new_plant = pw.PowerPlant(idnr,
name,
plant_country=COUNTRY_NAME,
cap = 0
if geolocation_source_string in geolocation_sources_mw:
geolocation_sources_mw[geolocation_source_string] += cap
else:
geolocation_sources_mw[geolocation_source_string] = cap
else:
try:
geolocation_sources_mw[u"Located, no source"] += capacity
except:
print(u" - Warning: plant {0} has no capacity".format(idnr))
# assign ID number
# have to use a hack for United Kingdom/GBR because we previously used an automated script
# original ID codes for GBR plants start with GBR, not WRI
if country == "United Kingdom":
idnr_full = pw.make_id("GBR", int(idnr))
else:
idnr_full = pw.make_id(SAVE_CODE, int(idnr))
# store the capacity by fuel for this plant
fuel_capacities = plant_fuel_capacities.get(idnr_full, {})
fuel_capacity = fuel_capacities.get(primary_fuel, 0)
if capacity:
fuel_capacity += capacity
for fuel_type in other_fuel:
if fuel_type not in fuel_capacities:
fuel_capacities[fuel_type] = 0
fuel_capacities[primary_fuel] = fuel_capacity
plant_fuel_capacities[idnr_full] = fuel_capacities
new_location = pw.LocationObject(latitude=0.0, longitude=0.0)
new_plant = pw.PowerPlant(plant_idnr=plant_idnr,
plant_name=name,
plant_country=COUNTRY_NAME,
plant_capacity=total_capacity,
plant_fuel=fuel_type_set,
plant_source=URL,
plant_location=new_location)
plants_dictionary[plant_idnr] = new_plant
print("Recording plant {0} with ID: {1}, capacity: {2}, fuel: {3}".
format(name, plant_idnr, total_capacity, fuel_type_set))
# next process this plant
name = pw.format_string(name_str)
plant_idnr = pw.make_id(SAVE_CODE, i)
capacity_list = [capacity]
year_built_list = [year_built]
fuel_type_set = fuel_type
else: # not a new plant, just a new line
if capacity_val:
capacity_list.append(capacity)
if year_built_val:
year_built_list.append(year_built)
if fuel_type and fuel_type not in fuel_type_set:
fuel_type_set.update(fuel_type)
# complete loop, add final plant
total_capacity = sum(capacity_list)
average_year_built = sum(year_built_list) / len(
try:
countries = pw.format_string(rv[countries_col])
country_list_iso2 = countries.split(";")
country_list = []
for iso2 in country_list_iso2:
country_list.append(iso2_to_country_names[iso2.strip()])
country = " ".join(country_list)
except:
print(
u"-Error: Can't read countries from string {0}; list: {1}.".format(
countries, country_list_iso2))
country = pw.NO_DATA_UNICODE
# assign ID number
idnr = pw.make_id(SAVE_CODE, ref)
new_location = pw.LocationObject(pw.NO_DATA_UNICODE, latitude, longitude)
new_plant = pw.PowerPlant(plant_idnr=idnr,
plant_name=name,
plant_country=country,
plant_location=new_location,
plant_coord_source=SOURCE_NAME,
plant_fuel=fuel,
plant_capacity=capacity,
plant_source=SOURCE_NAME,
plant_source_url=SOURCE_URL,
plant_owner=owner)
plants_dictionary[idnr] = new_plant
# report on plants read from file
print(u"Loaded {0} plants to database.".format(len(plants_dictionary)))
source = pw.format_string(row[source_col], None)
except:
print(u"-Error: Can't read data source for plant with name {0}".format(
name))
source = pw.NO_DATA_UNICODE
try:
data_date = (int(str(row[date_col])[0:4]))
except:
print(
u"-Error:Can't read reference date for plant with name {0}".format(
name))
data_date = pw.NO_DATA_NUMERIC
# assign ID number
idnr = pw.make_id(SAVE_CODE, i)
new_location = pw.LocationObject(pw.NO_DATA_UNICODE, latitude, longitude)
new_plant = pw.PowerPlant(plant_idnr=idnr,
plant_name=name,
plant_country=COUNTRY_NAME,
plant_owner=owner,
plant_cap_year=data_date,
plant_location=new_location,
plant_coord_source=geolocation_source,
plant_fuel=fuel,
plant_capacity=capacity,
plant_source=SOURCE_NAME_1,
plant_source_url=SOURCE_URL_1)
plants_dictionary[idnr] = new_plant
# use this for id incrementing in next file
source = pw.format_string(row[source_col], None)
except:
print(u"-Error: Can't read data source for plant with name {0}".format(
name))
source = pw.NO_DATA_UNICODE
try:
data_date = (int(str(row[date_col])[0:4]))
except:
print(
u"-Error:Can't read reference date for plant with name {0}".format(
name))
data_date = pw.NO_DATA_NUMERIC
# assign ID number
idnr = pw.make_id(SAVE_CODE, i)
new_location = pw.LocationObject(pw.NO_DATA_UNICODE, latitude, longitude)
new_plant = pw.PowerPlant(plant_idnr=idnr,
plant_name=name,
plant_country=COUNTRY_NAME,
plant_owner=owner,
plant_cap_year=data_date,
plant_location=new_location,
plant_coord_source=geolocation_source,
plant_primary_fuel=primary_fuel,
plant_capacity=capacity,
plant_source=source,
plant_source_url=SOURCE_URL_1)
plants_dictionary[idnr] = new_plant
# use this for id incrementing in next file
