Additional information: CARAM data api: http://carma.org/api/ """ import csv import sys, os sys.path.insert(0, os.pardir) import powerwatch as pw # params COUNTRY_NAME = u"GLOBAL" SOURCE_NAME = u"CARMA" SOURCE_URL = "http://carma.org/" SAVE_CODE = u"CARMA" RAW_FILE_NAME = pw.make_file_path( fileType="raw", subFolder=SAVE_CODE, filename="Full_CARMA_2009_Dataset_Power_Watch.csv") CSV_FILE_NAME = pw.make_file_path(fileType="src_csv", filename="carma_database.csv") SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin") YEAR_UPDATED = 2009 COLNAMES = ['plant.id', 'plant', 'lat', 'lon', 'iso3'] # optional raw file(s) download # note: API documentation says to specify "limit=0" to download entire dataset. # this doesn't work; it only downloads 2000 plants. # full dataset has about 51k plants, so specify 60k for limit to be safe. #URL = "http://carma.org/javascript/ajax-lister.php?type=plant&sort=carbon_present%20DESC&page=1&time=present&m1=world&m2=plant&m3=&m4=&export=make" #URL = "http://carma.org/api/1.1/searchPlants?raw=1&limit=20000" #FILES = {RAW_FILE_NAME: URL} #DOWNLOAD_FILES = pw.download(SOURCE_NAME, FILES)
import sys, os import json import wikimarkup import pandas as pd from pyquery import PyQuery sys.path.insert(0, os.pardir) import powerwatch as pw # params COUNTRY_NAME = u"China" # We are only gathering China coal data from this source SOURCE_NAME = u"SourceWatch" SAVE_CODE = u"SRCWT" RAW_FILE_NAME = pw.make_file_path( fileType="raw", subFolder=SAVE_CODE, filename="SourceWatch_China_coal_database.json") CSV_FILE_NAME = pw.make_file_path(fileType="src_csv", filename="sourcewatch_database.csv") SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin") SOURCE_URL = u"http://www.sourcewatch.org/index.php/Category:Existing_coal_plants_in_China" YEAR = 2017 # other params URL_BASE = "http://www.sourcewatch.org/api.php?" URL_END = "action=query&titles=Category:Existing_coal_plants_in_China&prop=revisions"\ +"&rvprop=content&format=json" # optional raw file(s) download URL = URL_BASE + URL_END FILES = {RAW_FILE_NAME: URL}
build_powerwatch.py Builds the PowerWatch database from various data sources. - Log build to POWERWATCH_BUILD_LOG_FILE - Use country and fuel information as specified in powerwatch.py """ import csv import time import argparse import sys, os sys.path.insert(0, os.pardir) import powerwatch as pw ### PARAMETERS ### COUNTRY_DATABASE_FILE = pw.make_file_path(fileType = "src_bin", filename = "COUNTRY-Database.bin") WRI_DATABASE_FILE = pw.make_file_path(fileType = "src_bin", filename = "WRI-Database.bin") GEO_DATABASE_FILE = pw.make_file_path(fileType = "src_bin", filename = "GEODB-Database.bin") CARMA_DATABASE_FILE = pw.make_file_path(fileType = "src_bin", filename = "CARMA-Database.bin") SOURCEWATCH_DATABASE_FILE = pw.make_file_path(fileType = "src_bin", filename = "SRCWT-Database.bin") POWERWATCH_CSV_SAVEFILE = pw.make_file_path(fileType = "output", filename = "powerwatch_data.csv") POWERWATCH_BUILD_LOG_FILE = pw.make_file_path(fileType = "output", filename = "powerwatch_build_log.txt") POWERWATCH_CSV_DUMPFILE = pw.make_file_path(fileType = "output", filename = "powerwatch_data_dump.csv") MINIMUM_CAPACITY_MW = 1 parser = argparse.ArgumentParser() parser.add_argument("--dump", help = "dump all the data", action="store_true") DATA_DUMP = True if parser.parse_args().dump else False # open log file f_log = open(POWERWATCH_BUILD_LOG_FILE,'a')
import argparse import csv import sys, os import xlrd from lxml import etree sys.path.insert(0, os.pardir) import powerwatch as pw # params COUNTRY_NAME = u"Global" SOURCE_NAME = u"Clean Development Mechanism" SOURCE_URL = u"https://cdm.unfccc.int" SAVE_CODE = u"CDMDB" RAW_FILE_NAME1 = pw.make_file_path(fileType="raw", subFolder=SAVE_CODE, filename="Database for PAs and PoAs.xlsx") RAW_FILE_NAME2 = pw.make_file_path(fileType="raw", subFolder=SAVE_CODE, filename="projectsLocationAll.xml") CSV_FILE_NAME = pw.make_file_path(fileType="src_csv", filename="CDMDB-database.csv") SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin") URL1 = "https://cdm.unfccc.int/Statistics/Public/files/Database%20for%20PAs%20and%20PoAs.xlsx" URL2 = "https://cdm.unfccc.int/Projects/MapApp/projectsLocationAll.xml" # project file specfications TAB_NAME = "CDM activities" COLNAMES = [ "CDM project reference number", "Unique project identifier (traceable with Google)",
from lxml import etree, html from dateutil.parser import parse as parse_date import csv import locale import sys, os sys.path.insert(0, os.pardir) import powerwatch as pw # params COUNTRY_NAME = u"Brazil" SOURCE_NAME = u"Agência Nacional de Energia Elétrica (Brazil)" SOURCE_URL = u"http://www2.aneel.gov.br/aplicacoes/capacidadebrasil/capacidadebrasil.cfm" SOURCE_YEAR = 2017 SAVE_CODE = u"BRA" RAW_FILE_NAME = pw.make_file_path(fileType="raw", subFolder=SAVE_CODE, filename="BRA_data.html") CSV_FILE_NAME = pw.make_file_path(fileType="src_csv", filename="database_BRA.csv") SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin") COORDINATE_FILE = pw.make_file_path(fileType="resource", subFolder=SAVE_CODE, filename="coordinates_BRA.csv") CAPACITY_CONVERSION_TO_MW = 0.001 # capacity values are given in kW in the raw data ENCODING = "ISO-8859-1" # set locale to Portuguese/Brazil locale.setlocale(locale.LC_ALL,'pt_BR') # download files if requested (large file; slow) DOWNLOAD_URL = u"http://www2.aneel.gov.br/aplicacoes/capacidadebrasil/GeracaoTipoFase.asp" POST_DATA = {'tipo':0,'fase':3} DOWNLOAD_FILES = pw.download('ANEEL B.I.G.',{RAW_FILE_NAME:DOWNLOAD_URL},POST_DATA) # define specialized fuel type interpreter
import datetime import xlrd import sys import os import json sys.path.insert(0, os.pardir) import powerwatch as pw # params COUNTRY_NAME = u"Argentina" SOURCE_NAME = u"Ministerio de Energía y Minería" SOURCE_URL = u"http://energia3.mecon.gov.ar/contenidos/archivos/Reorganizacion/informacion_del_mercado/publicaciones/mercado_electrico/estadisticosectorelectrico/2015/A1.POT_GEN_COMB_POR_CENTRAL_2015.xlsx" SAVE_CODE = u"ARG" RAW_FILE_NAME = pw.make_file_path( fileType="raw", subFolder=SAVE_CODE, filename="A1.POT_GEN_COMB_POR_CENTRAL_2015.xlsx") LOCATION_FILE_NAME = pw.make_file_path(fileType="resource", subFolder=SAVE_CODE, filename="locations_ARG.csv") COMMISSIONING_YEAR_FILE_NAME = pw.make_file_path( fileType="resource", subFolder=SAVE_CODE, filename="commissioning_years_ARG.csv") CSV_FILE_NAME = pw.make_file_path(fileType="src_csv", filename="argentina_database.csv") SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin") YEAR_OF_DATA = 2015 CAPACITY_CONVERSION_TO_MW = 0.001 # capacity values are given in kW in the raw data GENERATION_CONVERSION_TO_GWH = 0.001 # generation values are given in MWh in the raw data
PowerWatch build_database_USA.py Converts plant-level data from EIA-860 and EIA-923 to PowerWatch format. """ import xlrd import sys, os sys.path.insert(0, os.pardir) import powerwatch as pw # params COUNTRY_NAME = u"United States of America" SAVE_CODE = u"USA" RAW_FILE_NAME_860_2 = pw.make_file_path(fileType="raw", subFolder=SAVE_CODE, filename="2___Plant_Y2015.xlsx") RAW_FILE_NAME_860_3 = pw.make_file_path(fileType="raw", subFolder=SAVE_CODE, filename="3_1_Generator_Y2015.xlsx") RAW_FILE_NAME_923_2 = pw.make_file_path( fileType="raw", subFolder=SAVE_CODE, filename="EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx") CSV_FILE_NAME = pw.make_file_path(fileType="src_csv", filename="usa_database.csv") SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin") SOURCE = u"U.S. Energy Information Administration" SOURCE_URL = u"http://www.eia.gov/electricity/data/browser/" YEAR = 2015 GENERATION_CONVERSION_TO_GWH = 0.001 # generation values are given in MWh in the raw data
Unicode: Using unicodecsv, so we can read and write more easily to/from unicode. """ import xlrd import sys, os import re sys.path.insert(0, os.pardir) import powerwatch as pw # params COUNTRY_NAME = u"Finland" SOURCE_NAME = u"Energiavirasto" SOURCE_URL = u"http://www.energiavirasto.fi/en/voimalaitosrekisteri" SAVE_CODE = u"FIN" RAW_FILE_NAME = pw.make_file_path(fileType = "raw", subFolder = SAVE_CODE, filename = "finland_power_plants.xlsx") CSV_FILE_NAME = pw.make_file_path(fileType = "src_csv", filename = "finland_database.csv") SAVE_DIRECTORY = pw.make_file_path(fileType = "src_bin") DATASET_URL = "http://www.energiavirasto.fi/documents/10191/0/Energiaviraston+Voimalaitosrekisteri+010117.xlsx/fa2c9bd2-e436-4dbb-a2e5-074ecbf11d23" COLS = {"name":0, "owner":1, "capacity_max":17, "capacity_avg":18, "gen_type":7, "fuel_type":[20, 21, 22]} TAB_NAME = "English" pattern = r" -?\D?\d+.?" # space, optional hyphen or letter followed by 0-3 digits followed by any characters except newline def regex_match(name_1, name_2): if re.sub(pattern, "", name_1) == re.sub(pattern, "", name_2): return re.sub(pattern, "''", name_1) return False # optional raw file(s) download
Additional information: https://github.com/wri/powerwatch Issues: Requires an API key to retrieve data from Fusion Tables. """ import argparse import csv import sys, os sys.path.insert(0, os.pardir) import powerwatch as pw # params COUNTRY_NAME = u"GLOBAL" SOURCE_NAME = u"WRI" SAVE_CODE = u"WRI" CSV_FILE_NAME = pw.make_file_path(fileType="src_csv", filename="wri_database.csv") SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin") RAW_FILE_DIRECTORY = pw.make_file_path(fileType="raw", subFolder=SAVE_CODE) API_KEY_FILE = pw.make_file_path(fileType="resource", subFolder="api_keys", filename="fusion_tables_api_key.txt") # other parameters as needed URL_BASE = "https://www.googleapis.com/fusiontables/v2/query?alt=csv&sql=SELECT * FROM " # set up country dictionary (need this for fusion table keys) country_dictionary = pw.make_country_dictionary() # get API key with open(API_KEY_FILE, 'r') as f: API_KEY = f.readline()
Converts plant-level data from Arab Union of Electricity to PowerWatch format. Note: The AUE data is in .xls format. We need to use xlrd to read this. Note: xlrd decodes values to unicode as it reads them. """ import xlrd import sys, os sys.path.insert(0, os.pardir) import powerwatch as pw # params COUNTRY_NAME = 'Yemen' RAW_SOURCE = "AUE" SAVE_CODE = u"YEM" RAW_FILE_NAME = pw.make_file_path(fileType="raw", subFolder=RAW_SOURCE, filename="Manual Of Power Stations2016.xls") CSV_FILE_NAME = pw.make_file_path(fileType="src_csv", filename="yemen_database.csv") SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin") URL = "http://www.auptde.org/Article_Files/Manual%20Of%20Power%20Stations2016.xls" COLS = {'name': 0, 'capacity': 3, 'year_built': 4, 'fuel_type': 5} START_ROW = 4 END_ROW = 41 TAB_NUMBER = 0 # optional raw file download DOWNLOAD_FILES = pw.download(COUNTRY_NAME, {RAW_FILE_NAME: URL}) # set up fuel type thesaurus
import sys import os from zipfile import ZipFile import lxml.html as LH import xlrd sys.path.insert(0, os.pardir) import powerwatch as pw # params COUNTRY_NAME = u"India" SOURCE_NAME = u"Central Electricity Authority and REC Registry of India" SOURCE_URL = u"http://www.cea.nic.in/,https://www.recregistryindia.nic.in/" SAVE_CODE = u"IND" RAW_FILE_NAME_CEA = pw.make_file_path(fileType="raw", subFolder=SAVE_CODE, filename="database_11.zip") RAW_FILE_NAME_CEA_UZ = pw.make_file_path(fileType="raw", filename=SAVE_CODE) RAW_FILE_NAME_REC = pw.make_file_path( fileType="raw", subFolder=SAVE_CODE, filename="accredited_rec_generators.html") CSV_FILE_NAME = pw.make_file_path(fileType="src_csv", filename="database_IND.csv") SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin") LOCATION_FILE = pw.make_file_path(fileType="resource", subFolder=SAVE_CODE, filename="plant_locations_IND.csv") TAB_NAME = u"Data" DATA_YEAR = 2015
from zipfile import ZipFile from lxml import etree, html import sys, os sys.path.insert(0, os.pardir) sys.path.insert(0, os.path.join(os.pardir, os.pardir, os.pardir)) import powerwatch as pw # params COUNTRY_NAME = u"Brazil" SOURCE_NAME = u"Agência Nacional de Energia Elétrica (Brazil)" SOURCE_URL = u"http://sigel.aneel.gov.br/kmz.html" SAVE_CODE = u"BRA" RAW_FILE_NAME = pw.make_file_path(fileType="raw", subFolder=SAVE_CODE, filename="FILENAME.zip") CSV_FILE_NAME = "coordinates_BRA.csv" ENCODING = "UTF-8" # other parameters DATASETS = { 0: { "name": "UHE", "fuel": "Hydro" }, 1: { "name": "PCH", "fuel": "Hydro" }, 2: {
import xlrd import pyproj as pyproj sys.path.insert(0, os.pardir) import powerwatch as pw # params COUNTRY_NAME = u"United Kingdom" SOURCE_NAME = u"Department for Business Energy & Industrial Strategy" SOURCE_URL = u"https://www.gov.uk/government/collections/digest-of-uk-energy-statistics-dukes;https://www.gov.uk/government/collections/renewable-energy-planning-data" SAVE_CODE_GBR = u"GBR" SAVE_CODE_GEO = u"GEODB" SAVE_CODE_CARMA = u"CARMA" RAW_FILE_NAME_REPD = pw.make_file_path( fileType="raw", subFolder=SAVE_CODE_GBR, filename="2016_12_December_-_Public_Database_-_Dec_2016.csv") RAW_FILE_NAME_DUKES = pw.make_file_path(fileType="raw", subFolder=SAVE_CODE_GBR, filename="DUKES_5.10.xls") CSV_FILE_NAME = pw.make_file_path(fileType="src_csv", filename="gbr_database.csv") SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin") GEO_DATABASE_FILE = pw.make_file_path(fileType="src_bin", filename="GEODB-Database.bin") CARMA_DATABASE_FILE = pw.make_file_path(fileType="src_bin", filename="CARMA-Database.bin") REPD_YEAR = 2016 DUKES_YEAR = 2016 # other params
import csv import sys, os # import [other packages] sys.path.insert(0, os.pardir) import powerwatch as pw # params COUNTRY_NAME = u"NAME OF COUNTRY OR GLOBAL" SOURCE_NAME = u"NAME OF ORGANIZATION OR DATABASE[, OTHER ORGANIZATION OR DATABASE]" SOURCE_URL = u"PRIMARY URL" SAVE_CODE = u"3-LETTER SAVE CODE HERE (ISO CODE FOR COUNTRIES)" YEAR_POSTED = 2017 # Year of data posted on line RAW_FILE_NAME = pw.make_file_path(fileType="raw", subFolder=SAVE_CODE, filename="RAW FILE HERE") CSV_FILE_NAME = pw.make_file_path( fileType="src_csv", filename="database_{0}.csv".format(SAVE_CODE)) SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin") #LOCATION_FILE_NAME = pw.make_file_path(fileType="resource",subFolder=SAVE_CODE,filename="locations_{0}.csv".format(SAVE_CODE)) # other parameters as needed # specific functions here # def xyz(): # optional raw file(s) download # True if specified --download, otherwise False FILES = { RAW_FILE_NAME: URL
""" import csv import sys, os import xlrd sys.path.insert(0, os.pardir) import powerwatch as pw # params COUNTRY_NAME = u"Mexico" SOURCE_NAME = u"North American Cooperation on Energy Information and Comisión Reguladora de Energía" SAVE_CODE = u"MEX" RAW_FILE_NAME_1 = pw.make_file_path( fileType="raw", subFolder=SAVE_CODE, filename="PowerPlantsAllGE100MW_NorthAmerica_201606.xlsx") SOURCE_URL_1 = u"http://base.energia.gob.mx/nacei/Archivos/3%20P%C3%A1gina%20Datos%20de%20Infraestructura/Ingl%C3%A9s/PowerPlantsAllGE100MW_NorthAmerica_201606.xlsx" RAW_FILE_NAME_2 = pw.make_file_path( fileType="raw", subFolder=SAVE_CODE, filename="PowerPlantsRenewGE1MW_NorthAmerica_201606.xlsx") SOURCE_URL_2 = u"http://base.energia.gob.mx/nacei/Archivos/3%20P%C3%A1gina%20Datos%20de%20Infraestructura/Ingl%C3%A9s/PowerPlantsRenewGE1MW_NorthAmerica_201606.xlsx" RAW_FILE_NAME_3 = pw.make_file_path(fileType="raw", subFolder=SAVE_CODE, filename="1814.xlsx") SOURCE_URL_3 = u"http://www.cre.gob.mx/documento/1814.xlsx" CSV_FILE_NAME = pw.make_file_path(fileType="src_csv", filename="database_MEX.csv") SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin")
""" import csv import sys, os sys.path.insert(0, os.pardir) import powerwatch as pw # params COUNTRY_NAME = u"Chile" SOURCE_NAME = u"Energía Abierta" SOURCE_URL = u"http://energiaabierta.cl/electricidad/" SAVE_CODE = u"CHL" YEAR_POSTED = 2016 # Year of data posted on line RAW_FILE_NAME = pw.make_file_path(fileType='raw', subFolder=SAVE_CODE, filename="FILENAME") CSV_FILE_NAME = pw.make_file_path( fileType="src_csv", filename="database_{0}.csv".format(SAVE_CODE)) SAVE_DIRECTORY = pw.make_file_path(fileType="src_bin") LOCATION_FILE_NAME = pw.make_file_path( fileType="resource", subFolder=SAVE_CODE, filename="locations_{0}.csv".format(SAVE_CODE)) # other parameters URL_BASE = "http://datos.energiaabierta.cl/rest/datastreams/NUMBER/data.csv" DATASETS = [{ "number": "215392", "fuel": "Thermal",