Additional information: CARAM data api: http://carma.org/api/
"""
import csv
import sys, os
sys.path.insert(0, os.pardir)
import powerwatch as pw
# params
COUNTRY_NAME = u"GLOBAL"
SOURCE_NAME = u"CARMA"
SOURCE_URL = "http://carma.org/"
SAVE_CODE = u"CARMA"
RAW_FILE_NAME = pw.make_file_path(
fileType="raw",
subFolder=SAVE_CODE,
filename="Full_CARMA_2009_Dataset_Power_Watch.csv")
CSV_FILE_NAME = pw.make_file_path(fileType="src_csv",
filename="carma_database.csv")
SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin")
YEAR_UPDATED = 2009
COLNAMES = ['plant.id', 'plant', 'lat', 'lon', 'iso3']
# optional raw file(s) download
# note: API documentation says to specify "limit=0" to download entire dataset.
# this doesn't work; it only downloads 2000 plants.
# full dataset has about 51k plants, so specify 60k for limit to be safe.
#URL = "http://carma.org/javascript/ajax-lister.php?type=plant&sort=carbon_present%20DESC&page=1&time=present&m1=world&m2=plant&m3=&m4=&export=make"
#URL = "http://carma.org/api/1.1/searchPlants?raw=1&limit=20000"
#FILES = {RAW_FILE_NAME: URL}
#DOWNLOAD_FILES = pw.download(SOURCE_NAME, FILES)
import sys, os
import json
import wikimarkup
import pandas as pd
from pyquery import PyQuery
sys.path.insert(0, os.pardir)
import powerwatch as pw
# params
COUNTRY_NAME = u"China" # We are only gathering China coal data from this source
SOURCE_NAME = u"SourceWatch"
SAVE_CODE = u"SRCWT"
RAW_FILE_NAME = pw.make_file_path(
fileType="raw",
subFolder=SAVE_CODE,
filename="SourceWatch_China_coal_database.json")
CSV_FILE_NAME = pw.make_file_path(fileType="src_csv",
filename="sourcewatch_database.csv")
SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin")
SOURCE_URL = u"http://www.sourcewatch.org/index.php/Category:Existing_coal_plants_in_China"
YEAR = 2017
# other params
URL_BASE = "http://www.sourcewatch.org/api.php?"
URL_END = "action=query&titles=Category:Existing_coal_plants_in_China&prop=revisions"\
+"&rvprop=content&format=json"
# optional raw file(s) download
URL = URL_BASE + URL_END
FILES = {RAW_FILE_NAME: URL}
build_powerwatch.py
Builds the PowerWatch database from various data sources.
- Log build to POWERWATCH_BUILD_LOG_FILE
- Use country and fuel information as specified in powerwatch.py
"""
import csv
import time
import argparse
import sys, os
sys.path.insert(0, os.pardir)
import powerwatch as pw
### PARAMETERS ###
COUNTRY_DATABASE_FILE = pw.make_file_path(fileType = "src_bin", filename = "COUNTRY-Database.bin")
WRI_DATABASE_FILE = pw.make_file_path(fileType = "src_bin", filename = "WRI-Database.bin")
GEO_DATABASE_FILE = pw.make_file_path(fileType = "src_bin", filename = "GEODB-Database.bin")
CARMA_DATABASE_FILE = pw.make_file_path(fileType = "src_bin", filename = "CARMA-Database.bin")
SOURCEWATCH_DATABASE_FILE = pw.make_file_path(fileType = "src_bin", filename = "SRCWT-Database.bin")
POWERWATCH_CSV_SAVEFILE = pw.make_file_path(fileType = "output", filename = "powerwatch_data.csv")
POWERWATCH_BUILD_LOG_FILE = pw.make_file_path(fileType = "output", filename = "powerwatch_build_log.txt")
POWERWATCH_CSV_DUMPFILE = pw.make_file_path(fileType = "output", filename = "powerwatch_data_dump.csv")
MINIMUM_CAPACITY_MW = 1
parser = argparse.ArgumentParser()
parser.add_argument("--dump", help = "dump all the data", action="store_true")
DATA_DUMP = True if parser.parse_args().dump else False
# open log file
f_log = open(POWERWATCH_BUILD_LOG_FILE,'a')
import argparse
import csv
import sys, os
import xlrd
from lxml import etree
sys.path.insert(0, os.pardir)
import powerwatch as pw
# params
COUNTRY_NAME = u"Global"
SOURCE_NAME = u"Clean Development Mechanism"
SOURCE_URL = u"https://cdm.unfccc.int"
SAVE_CODE = u"CDMDB"
RAW_FILE_NAME1 = pw.make_file_path(fileType="raw",
subFolder=SAVE_CODE,
filename="Database for PAs and PoAs.xlsx")
RAW_FILE_NAME2 = pw.make_file_path(fileType="raw",
subFolder=SAVE_CODE,
filename="projectsLocationAll.xml")
CSV_FILE_NAME = pw.make_file_path(fileType="src_csv",
filename="CDMDB-database.csv")
SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin")
URL1 = "https://cdm.unfccc.int/Statistics/Public/files/Database%20for%20PAs%20and%20PoAs.xlsx"
URL2 = "https://cdm.unfccc.int/Projects/MapApp/projectsLocationAll.xml"
# project file specfications
TAB_NAME = "CDM activities"
COLNAMES = [
"CDM project reference number",
"Unique project identifier (traceable with Google)",
from lxml import etree, html
from dateutil.parser import parse as parse_date
import csv
import locale
import sys, os
sys.path.insert(0, os.pardir)
import powerwatch as pw
# params
COUNTRY_NAME = u"Brazil"
SOURCE_NAME = u"Agência Nacional de Energia Elétrica (Brazil)"
SOURCE_URL = u"http://www2.aneel.gov.br/aplicacoes/capacidadebrasil/capacidadebrasil.cfm"
SOURCE_YEAR = 2017
SAVE_CODE = u"BRA"
RAW_FILE_NAME = pw.make_file_path(fileType="raw", subFolder=SAVE_CODE, filename="BRA_data.html")
CSV_FILE_NAME = pw.make_file_path(fileType="src_csv", filename="database_BRA.csv")
SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin")
COORDINATE_FILE = pw.make_file_path(fileType="resource", subFolder=SAVE_CODE, filename="coordinates_BRA.csv")
CAPACITY_CONVERSION_TO_MW = 0.001 # capacity values are given in kW in the raw data
ENCODING = "ISO-8859-1"
# set locale to Portuguese/Brazil
locale.setlocale(locale.LC_ALL,'pt_BR')
# download files if requested (large file; slow)
DOWNLOAD_URL = u"http://www2.aneel.gov.br/aplicacoes/capacidadebrasil/GeracaoTipoFase.asp"
POST_DATA = {'tipo':0,'fase':3}
DOWNLOAD_FILES = pw.download('ANEEL B.I.G.',{RAW_FILE_NAME:DOWNLOAD_URL},POST_DATA)
# define specialized fuel type interpreter
import datetime
import xlrd
import sys
import os
import json
sys.path.insert(0, os.pardir)
import powerwatch as pw
# params
COUNTRY_NAME = u"Argentina"
SOURCE_NAME = u"Ministerio de Energía y Minería"
SOURCE_URL = u"http://energia3.mecon.gov.ar/contenidos/archivos/Reorganizacion/informacion_del_mercado/publicaciones/mercado_electrico/estadisticosectorelectrico/2015/A1.POT_GEN_COMB_POR_CENTRAL_2015.xlsx"
SAVE_CODE = u"ARG"
RAW_FILE_NAME = pw.make_file_path(
fileType="raw",
subFolder=SAVE_CODE,
filename="A1.POT_GEN_COMB_POR_CENTRAL_2015.xlsx")
LOCATION_FILE_NAME = pw.make_file_path(fileType="resource",
subFolder=SAVE_CODE,
filename="locations_ARG.csv")
COMMISSIONING_YEAR_FILE_NAME = pw.make_file_path(
fileType="resource",
subFolder=SAVE_CODE,
filename="commissioning_years_ARG.csv")
CSV_FILE_NAME = pw.make_file_path(fileType="src_csv",
filename="argentina_database.csv")
SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin")
YEAR_OF_DATA = 2015
CAPACITY_CONVERSION_TO_MW = 0.001 # capacity values are given in kW in the raw data
GENERATION_CONVERSION_TO_GWH = 0.001 # generation values are given in MWh in the raw data
PowerWatch
build_database_USA.py
Converts plant-level data from EIA-860 and EIA-923 to PowerWatch format.
"""
import xlrd
import sys, os
sys.path.insert(0, os.pardir)
import powerwatch as pw
# params
COUNTRY_NAME = u"United States of America"
SAVE_CODE = u"USA"
RAW_FILE_NAME_860_2 = pw.make_file_path(fileType="raw",
subFolder=SAVE_CODE,
filename="2___Plant_Y2015.xlsx")
RAW_FILE_NAME_860_3 = pw.make_file_path(fileType="raw",
subFolder=SAVE_CODE,
filename="3_1_Generator_Y2015.xlsx")
RAW_FILE_NAME_923_2 = pw.make_file_path(
fileType="raw",
subFolder=SAVE_CODE,
filename="EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx")
CSV_FILE_NAME = pw.make_file_path(fileType="src_csv",
filename="usa_database.csv")
SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin")
SOURCE = u"U.S. Energy Information Administration"
SOURCE_URL = u"http://www.eia.gov/electricity/data/browser/"
YEAR = 2015
GENERATION_CONVERSION_TO_GWH = 0.001 # generation values are given in MWh in the raw data
Unicode: Using unicodecsv, so we can read and write more easily to/from unicode.
"""
import xlrd
import sys, os
import re
sys.path.insert(0, os.pardir)
import powerwatch as pw
# params
COUNTRY_NAME = u"Finland"
SOURCE_NAME = u"Energiavirasto"
SOURCE_URL = u"http://www.energiavirasto.fi/en/voimalaitosrekisteri"
SAVE_CODE = u"FIN"
RAW_FILE_NAME = pw.make_file_path(fileType = "raw", subFolder = SAVE_CODE, filename = "finland_power_plants.xlsx")
CSV_FILE_NAME = pw.make_file_path(fileType = "src_csv", filename = "finland_database.csv")
SAVE_DIRECTORY = pw.make_file_path(fileType = "src_bin")
DATASET_URL = "http://www.energiavirasto.fi/documents/10191/0/Energiaviraston+Voimalaitosrekisteri+010117.xlsx/fa2c9bd2-e436-4dbb-a2e5-074ecbf11d23"
COLS = {"name":0, "owner":1, "capacity_max":17, "capacity_avg":18,
"gen_type":7, "fuel_type":[20, 21, 22]}
TAB_NAME = "English"
pattern = r" -?\D?\d+.?" # space, optional hyphen or letter followed by 0-3 digits followed by any characters except newline
def regex_match(name_1, name_2):
if re.sub(pattern, "", name_1) == re.sub(pattern, "", name_2):
return re.sub(pattern, "''", name_1)
return False
# optional raw file(s) download
Additional information: https://github.com/wri/powerwatch
Issues: Requires an API key to retrieve data from Fusion Tables.
"""
import argparse
import csv
import sys, os
sys.path.insert(0, os.pardir)
import powerwatch as pw
# params
COUNTRY_NAME = u"GLOBAL"
SOURCE_NAME = u"WRI"
SAVE_CODE = u"WRI"
CSV_FILE_NAME = pw.make_file_path(fileType="src_csv",
filename="wri_database.csv")
SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin")
RAW_FILE_DIRECTORY = pw.make_file_path(fileType="raw", subFolder=SAVE_CODE)
API_KEY_FILE = pw.make_file_path(fileType="resource",
subFolder="api_keys",
filename="fusion_tables_api_key.txt")
# other parameters as needed
URL_BASE = "https://www.googleapis.com/fusiontables/v2/query?alt=csv&sql=SELECT * FROM "
# set up country dictionary (need this for fusion table keys)
country_dictionary = pw.make_country_dictionary()
# get API key
with open(API_KEY_FILE, 'r') as f:
API_KEY = f.readline()
Converts plant-level data from Arab Union of Electricity to PowerWatch format.
Note: The AUE data is in .xls format. We need to use xlrd to read this.
Note: xlrd decodes values to unicode as it reads them.
"""
import xlrd
import sys, os
sys.path.insert(0, os.pardir)
import powerwatch as pw
# params
COUNTRY_NAME = 'Yemen'
RAW_SOURCE = "AUE"
SAVE_CODE = u"YEM"
RAW_FILE_NAME = pw.make_file_path(fileType="raw",
subFolder=RAW_SOURCE,
filename="Manual Of Power Stations2016.xls")
CSV_FILE_NAME = pw.make_file_path(fileType="src_csv",
filename="yemen_database.csv")
SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin")
URL = "http://www.auptde.org/Article_Files/Manual%20Of%20Power%20Stations2016.xls"
COLS = {'name': 0, 'capacity': 3, 'year_built': 4, 'fuel_type': 5}
START_ROW = 4
END_ROW = 41
TAB_NUMBER = 0
# optional raw file download
DOWNLOAD_FILES = pw.download(COUNTRY_NAME, {RAW_FILE_NAME: URL})
# set up fuel type thesaurus
import sys
import os
from zipfile import ZipFile
import lxml.html as LH
import xlrd
sys.path.insert(0, os.pardir)
import powerwatch as pw
# params
COUNTRY_NAME = u"India"
SOURCE_NAME = u"Central Electricity Authority and REC Registry of India"
SOURCE_URL = u"http://www.cea.nic.in/,https://www.recregistryindia.nic.in/"
SAVE_CODE = u"IND"
RAW_FILE_NAME_CEA = pw.make_file_path(fileType="raw",
subFolder=SAVE_CODE,
filename="database_11.zip")
RAW_FILE_NAME_CEA_UZ = pw.make_file_path(fileType="raw", filename=SAVE_CODE)
RAW_FILE_NAME_REC = pw.make_file_path(
fileType="raw",
subFolder=SAVE_CODE,
filename="accredited_rec_generators.html")
CSV_FILE_NAME = pw.make_file_path(fileType="src_csv",
filename="database_IND.csv")
SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin")
LOCATION_FILE = pw.make_file_path(fileType="resource",
subFolder=SAVE_CODE,
filename="plant_locations_IND.csv")
TAB_NAME = u"Data"
DATA_YEAR = 2015
from zipfile import ZipFile
from lxml import etree, html
import sys, os
sys.path.insert(0, os.pardir)
sys.path.insert(0, os.path.join(os.pardir, os.pardir, os.pardir))
import powerwatch as pw
# params
COUNTRY_NAME = u"Brazil"
SOURCE_NAME = u"Agência Nacional de Energia Elétrica (Brazil)"
SOURCE_URL = u"http://sigel.aneel.gov.br/kmz.html"
SAVE_CODE = u"BRA"
RAW_FILE_NAME = pw.make_file_path(fileType="raw",
subFolder=SAVE_CODE,
filename="FILENAME.zip")
CSV_FILE_NAME = "coordinates_BRA.csv"
ENCODING = "UTF-8"
# other parameters
DATASETS = {
0: {
"name": "UHE",
"fuel": "Hydro"
},
1: {
"name": "PCH",
"fuel": "Hydro"
},
2: {
import xlrd
import pyproj as pyproj
sys.path.insert(0, os.pardir)
import powerwatch as pw
# params
COUNTRY_NAME = u"United Kingdom"
SOURCE_NAME = u"Department for Business Energy & Industrial Strategy"
SOURCE_URL = u"https://www.gov.uk/government/collections/digest-of-uk-energy-statistics-dukes;https://www.gov.uk/government/collections/renewable-energy-planning-data"
SAVE_CODE_GBR = u"GBR"
SAVE_CODE_GEO = u"GEODB"
SAVE_CODE_CARMA = u"CARMA"
RAW_FILE_NAME_REPD = pw.make_file_path(
fileType="raw",
subFolder=SAVE_CODE_GBR,
filename="2016_12_December_-_Public_Database_-_Dec_2016.csv")
RAW_FILE_NAME_DUKES = pw.make_file_path(fileType="raw",
subFolder=SAVE_CODE_GBR,
filename="DUKES_5.10.xls")
CSV_FILE_NAME = pw.make_file_path(fileType="src_csv",
filename="gbr_database.csv")
SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin")
GEO_DATABASE_FILE = pw.make_file_path(fileType="src_bin",
filename="GEODB-Database.bin")
CARMA_DATABASE_FILE = pw.make_file_path(fileType="src_bin",
filename="CARMA-Database.bin")
REPD_YEAR = 2016
DUKES_YEAR = 2016
# other params
import csv
import sys, os
# import [other packages]
sys.path.insert(0, os.pardir)
import powerwatch as pw
# params
COUNTRY_NAME = u"NAME OF COUNTRY OR GLOBAL"
SOURCE_NAME = u"NAME OF ORGANIZATION OR DATABASE[, OTHER ORGANIZATION OR DATABASE]"
SOURCE_URL = u"PRIMARY URL"
SAVE_CODE = u"3-LETTER SAVE CODE HERE (ISO CODE FOR COUNTRIES)"
YEAR_POSTED = 2017 # Year of data posted on line
RAW_FILE_NAME = pw.make_file_path(fileType="raw",
subFolder=SAVE_CODE,
filename="RAW FILE HERE")
CSV_FILE_NAME = pw.make_file_path(
fileType="src_csv", filename="database_{0}.csv".format(SAVE_CODE))
SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin")
#LOCATION_FILE_NAME = pw.make_file_path(fileType="resource",subFolder=SAVE_CODE,filename="locations_{0}.csv".format(SAVE_CODE))
# other parameters as needed
# specific functions here
# def xyz():
# optional raw file(s) download
# True if specified --download, otherwise False
FILES = {
RAW_FILE_NAME: URL
"""
import csv
import sys, os
import xlrd
sys.path.insert(0, os.pardir)
import powerwatch as pw
# params
COUNTRY_NAME = u"Mexico"
SOURCE_NAME = u"North American Cooperation on Energy Information and Comisión Reguladora de Energía"
SAVE_CODE = u"MEX"
RAW_FILE_NAME_1 = pw.make_file_path(
fileType="raw",
subFolder=SAVE_CODE,
filename="PowerPlantsAllGE100MW_NorthAmerica_201606.xlsx")
SOURCE_URL_1 = u"http://base.energia.gob.mx/nacei/Archivos/3%20P%C3%A1gina%20Datos%20de%20Infraestructura/Ingl%C3%A9s/PowerPlantsAllGE100MW_NorthAmerica_201606.xlsx"
RAW_FILE_NAME_2 = pw.make_file_path(
fileType="raw",
subFolder=SAVE_CODE,
filename="PowerPlantsRenewGE1MW_NorthAmerica_201606.xlsx")
SOURCE_URL_2 = u"http://base.energia.gob.mx/nacei/Archivos/3%20P%C3%A1gina%20Datos%20de%20Infraestructura/Ingl%C3%A9s/PowerPlantsRenewGE1MW_NorthAmerica_201606.xlsx"
RAW_FILE_NAME_3 = pw.make_file_path(fileType="raw",
subFolder=SAVE_CODE,
filename="1814.xlsx")
SOURCE_URL_3 = u"http://www.cre.gob.mx/documento/1814.xlsx"
CSV_FILE_NAME = pw.make_file_path(fileType="src_csv",
filename="database_MEX.csv")
SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin")
"""
import csv
import sys, os
sys.path.insert(0, os.pardir)
import powerwatch as pw
# params
COUNTRY_NAME = u"Chile"
SOURCE_NAME = u"Energía Abierta"
SOURCE_URL = u"http://energiaabierta.cl/electricidad/"
SAVE_CODE = u"CHL"
YEAR_POSTED = 2016 # Year of data posted on line
RAW_FILE_NAME = pw.make_file_path(fileType='raw',
subFolder=SAVE_CODE,
filename="FILENAME")
CSV_FILE_NAME = pw.make_file_path(
fileType="src_csv", filename="database_{0}.csv".format(SAVE_CODE))
SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin")
LOCATION_FILE_NAME = pw.make_file_path(
fileType="resource",
subFolder=SAVE_CODE,
filename="locations_{0}.csv".format(SAVE_CODE))
# other parameters
URL_BASE = "http://datos.energiaabierta.cl/rest/datastreams/NUMBER/data.csv"
DATASETS = [{
"number": "215392",
"fuel": "Thermal",
