def test_get_response_soup(self):
try:
response_soup = web.get_response_soup('ES',
'https://www.google.es')
self.assertIsNotNone(response_soup)
except ParserException as ex:
self.fail("assert_country_code() raised Exception unexpectedly!")
def test_get_response_soup(self):
try:
response_soup = web.get_response_soup("ES",
"https://www.google.es")
self.assertIsNotNone(response_soup)
except ParserException as ex:
self.fail("assert_zone_key() raised Exception unexpectedly!")
def fetch_production(country_code='IN-CT', session=None):
"""Fetch Chhattisgarh production"""
countrycode.assert_country_code(country_code, 'IN-CT')
html = web.get_response_soup(country_code, 'http://117.239.199.203/csptcl/GEN.aspx', session)
india_date_time = IN.read_datetime_from_span_id(html, 'L37', 'hh:m DD-MM-YY')
korba_east_value = IN.read_value_from_span_id(html, 'L7')
korba_west_value = IN.read_value_from_span_id(html, 'L13')
dsmp_value = IN.read_value_from_span_id(html, 'L16')
marwa_value = IN.read_value_from_span_id(html, 'L23')
coal_value = round(korba_east_value + korba_west_value + dsmp_value + marwa_value, 2)
bango_value = IN.read_value_from_span_id(html, 'L20')
data = {
'countryCode': country_code,
'datetime': india_date_time.datetime,
'production': {
'coal': coal_value,
'hydro': bango_value
},
'source': 'cspc.co.in',
}
return data
def fetch_production(country_code='IN-CT', session=None):
"""Fetch Chhattisgarh production"""
countrycode.assert_country_code(country_code, 'IN-CT')
html = web.get_response_soup(country_code, 'http://117.239.199.203/csptcl/GEN.aspx', session)
india_date_time = IN.read_datetime_from_span_id(html, 'L37', 'hh:m DD-MM-YY')
korba_east_value = IN.read_value_from_span_id(html, 'L7')
korba_west_value = IN.read_value_from_span_id(html, 'L13')
dsmp_value = IN.read_value_from_span_id(html, 'L16')
marwa_value = IN.read_value_from_span_id(html, 'L23')
coal_value = round(korba_east_value + korba_west_value + dsmp_value + marwa_value, 2)
bango_value = IN.read_value_from_span_id(html, 'L20')
data = {
'countryCode': country_code,
'datetime': india_date_time.datetime,
'production': {
'coal': coal_value,
'hydro': bango_value
},
'source': 'cspc.co.in',
}
return data
def fetch_production(country_code='IN-DL', session=None):
"""Fetch Delhi production"""
countrycode.assert_country_code(country_code, 'IN-DL')
html = web.get_response_soup(
country_code, 'http://www.delhisldc.org/Redirect.aspx?Loc=0804',
session)
india_date_string = IN.read_text_from_span_id(
html, 'ContentPlaceHolder3_ddgenco')
india_date_time = IN.read_datetime_with_only_time(india_date_string,
'HH:mm:ss')
prod_table = html.find("table", {"id": "ContentPlaceHolder3_dgenco"})
prod_rows = prod_table.findAll('tr')
# BTPS https://en.wikipedia.org/wiki/Badarpur_Thermal_Power_Station
btps = read_value(prod_rows[1])
# CCGT https://en.wikipedia.org/wiki/Pragati-III_Combined_Cycle_Power_Plant = Pragati-3
ccgt = read_value(prod_rows[2])
# DMSWSL (Delhi Municipal Solid Waste Solutions Limited): Garbage-to-electricity
dmswsl = read_value(prod_rows[3])
# EDWPL (East Delhi Waste Processing Company Limited): Garbage-to-electricity
edwpl = read_value(prod_rows[4])
# GT (Gas Turbine) https://en.wikipedia.org/wiki/IPGCL_Gas_Turbine_Power_Station
gt = read_value(prod_rows[5])
# Pragati https://en.wikipedia.org/wiki/Pragati-I_Combined_Cycle_Gas_Power_Station = Pragati-1
pragati = read_value(prod_rows[6])
# TOWMP (Timarpur Okhla Waste Management Company Pvt. Ltd.): Garbage-to-electricity
towmp = read_value(prod_rows[7])
# Coal production
coal = btps
# Gas production
gas = ccgt + pragati + gt
# Unknown production
garbage = dmswsl + edwpl + towmp
data = {
'countryCode': country_code,
'datetime': india_date_time.datetime,
'production': {
'coal': coal,
'gas': gas,
'biomass': garbage
},
'source': 'delhisldc.org',
}
return data
def fetch_production(country_code='IN-DL', session=None):
"""Fetch Delhi production"""
countrycode.assert_country_code(country_code, 'IN-DL')
html = web.get_response_soup(
country_code, 'http://www.delhisldc.org/Redirect.aspx?Loc=0804',
session)
india_date_string = IN.read_text_from_span_id(
html, 'ContentPlaceHolder3_ddgenco')
india_date_time = IN.read_datetime_with_only_time(india_date_string,
'HH:mm:ss')
prod_table = html.find("table", {"id": "ContentPlaceHolder3_dgenco"})
prod_rows = prod_table.findAll('tr')
# BTPS https://en.wikipedia.org/wiki/Badarpur_Thermal_Power_Station
btps = read_value(prod_rows[1])
# CCGT https://en.wikipedia.org/wiki/Pragati-III_Combined_Cycle_Power_Plant
ccgt = read_value(prod_rows[2])
# DMSWSL Unknown
dmswsl = read_value(prod_rows[3])
# EDWPL Unknown
edwpl = read_value(prod_rows[4])
# GT Unknown
gt = read_value(prod_rows[5])
# Pragati
pragati = read_value(prod_rows[6])
# TOWMP Waste?
towmp = read_value(prod_rows[7])
# Coal production
coal = btps
# Gas production
gas = ccgt + pragati
# Unknown production
unknown_value = dmswsl + edwpl + gt + towmp
data = {
'countryCode': country_code,
'datetime': india_date_time.datetime,
'production': {
'coal': coal,
'gas': gas,
'unknown': unknown_value
},
'source': 'delhisldc.org',
}
return data
def fetch_production(country_code='IN-DL', session=None):
"""Fetch Delhi production"""
countrycode.assert_country_code(country_code, 'IN-DL')
html = web.get_response_soup(country_code, 'http://www.delhisldc.org/Redirect.aspx?Loc=0804', session)
india_date_string = IN.read_text_from_span_id(html, 'ContentPlaceHolder3_ddgenco')
india_date_time = IN.read_datetime_with_only_time(india_date_string, 'HH:mm:ss')
prod_table = html.find("table", {"id": "ContentPlaceHolder3_dgenco"})
prod_rows = prod_table.findAll('tr')
# BTPS https://en.wikipedia.org/wiki/Badarpur_Thermal_Power_Station
btps = read_value(prod_rows[1])
# CCGT https://en.wikipedia.org/wiki/Pragati-III_Combined_Cycle_Power_Plant
ccgt = read_value(prod_rows[2])
# DMSWSL Unknown
dmswsl = read_value(prod_rows[3])
# EDWPL Unknown
edwpl = read_value(prod_rows[4])
# GT Unknown
gt = read_value(prod_rows[5])
# Pragati
pragati = read_value(prod_rows[6])
# TOWMP Waste?
towmp = read_value(prod_rows[7])
# Coal production
coal = btps
# Gas production
gas = ccgt + pragati
# Unknown production
unknown_value = dmswsl + edwpl + gt + towmp
data = {
'countryCode': country_code,
'datetime': india_date_time.datetime,
'production': {
'coal': coal,
'gas': gas,
'unknown': unknown_value
},
'source': 'delhisldc.org',
}
return data
def fetch_consumption(country_code='IN-DL', session=None):
"""Fetch Delhi consumption"""
countrycode.assert_country_code(country_code, 'IN-DL')
html = web.get_response_soup(country_code, 'http://www.delhisldc.org/Redirect.aspx', session)
india_date_time = IN.read_datetime_from_span_id(html, 'DynamicData1_LblDate', 'DD-MMM-YYYY hh:mm:ss A')
demand_value = IN.read_value_from_span_id(html, 'DynamicData1_LblLoad')
data = {
'countryCode': country_code,
'datetime': india_date_time.datetime,
'consumption': demand_value,
'source': 'delhisldc.org'
}
return data
def fetch_consumption(country_code='IN-CT', session=None):
"""Fetch Chhattisgarh consumption"""
countrycode.assert_country_code(country_code, 'IN-CT')
html = web.get_response_soup(country_code, 'http://117.239.199.203/csptcl/GEN.aspx', session)
india_date_time = IN.read_datetime_from_span_id(html, 'L37', 'hh:m DD-MM-YY')
demand_value = IN.read_value_from_span_id(html, 'L29')
data = {
'countryCode': country_code,
'datetime': india_date_time.datetime,
'consumption': demand_value,
'source': 'cspc.co.in'
}
return data
def fetch_consumption(country_code='IN-KA', session=None):
"""Fetch Karnataka consumption"""
countrycode.assert_country_code(country_code, 'IN-KA')
html = web.get_response_soup(country_code, 'http://kptclsldc.com/Default.aspx', session)
india_date_time = IN.read_datetime_from_span_id(html, 'Label6', 'DD/MM/YYYY HH:mm')
demand_value = IN.read_value_from_span_id(html, 'Label5')
data = {
'countryCode': country_code,
'datetime': india_date_time.datetime,
'consumption': demand_value,
'source': 'kptclsldc.com'
}
return data
def fetch_consumption(country_code='IN-CT', session=None):
"""Fetch Chhattisgarh consumption"""
countrycode.assert_country_code(country_code, 'IN-CT')
html = web.get_response_soup(country_code, 'http://117.239.199.203/csptcl/GEN.aspx', session)
india_date_time = IN.read_datetime_from_span_id(html, 'L37', 'hh:m DD-MM-YY')
demand_value = IN.read_value_from_span_id(html, 'L29')
data = {
'countryCode': country_code,
'datetime': india_date_time.datetime,
'consumption': demand_value,
'source': 'cspc.co.in'
}
return data
def fetch_consumption(country_code='IN-KA', session=None):
"""Fetch Karnataka consumption"""
countrycode.assert_country_code(country_code, 'IN-KA')
html = web.get_response_soup(country_code, 'http://kptclsldc.com/Default.aspx', session)
india_date_time = india.read_datetime_from_span_id(html, 'Label6', 'DD/MM/YYYY HH:mm')
demand_value = india.read_value_from_span_id(html, 'Label5')
data = {
'countryCode': country_code,
'datetime': india_date_time.datetime,
'consumption': demand_value,
'source': 'kptclsldc.com'
}
return data
def fetch_consumption(country_code='IN-DL', session=None):
"""Fetch Delhi consumption"""
countrycode.assert_country_code(country_code, 'IN-DL')
html = web.get_response_soup(country_code,
'http://www.delhisldc.org/Redirect.aspx',
session)
india_date_time = IN.read_datetime_from_span_id(html,
'DynamicData1_LblDate',
'DD-MMM-YYYY hh:mm:ss A')
demand_value = IN.read_value_from_span_id(html, 'DynamicData1_LblLoad')
data = {
'countryCode': country_code,
'datetime': india_date_time.datetime,
'consumption': demand_value,
'source': 'delhisldc.org'
}
return data
def test_get_response_soup(self):
try:
response_soup = web.get_response_soup('ES', 'https://www.google.es')
self.assertIsNotNone(response_soup)
except ParserException as ex:
self.fail("assert_country_code() raised Exception unexpectedly!")
def fetch_production(country_code='IN-KA', session=None):
"""Fetch Karnataka production"""
countrycode.assert_country_code(country_code, 'IN-KA')
html = web.get_response_soup(country_code, 'http://kptclsldc.com/StateGen.aspx', session)
india_date_time = IN.read_datetime_from_span_id(html, 'lbldate', 'M/D/YYYY h:mm:ss A')
# RTPS Production: https://en.wikipedia.org/wiki/Raichur_Thermal_Power_Station
rtps_value = IN.read_value_from_span_id(html, 'lblrtptot')
# BTPS Production: https://en.wikipedia.org/wiki/Bellary_Thermal_Power_station
btps_value = IN.read_value_from_span_id(html, 'lblbtptot')
# YTPS Production: https://en.wikipedia.org/wiki/Yermarus_Thermal_Power_Station
ytps_value = IN.read_value_from_span_id(html, 'ytptot')
# UPCL Production: https://en.wikipedia.org/wiki/Udupi_Power_Plant
upcl_value = IN.read_value_from_span_id(html, 'lblupctot')
# JINDAl Production
jindal_value = IN.read_value_from_span_id(html, 'lbljintot')
# Coal Production
coal_value = rtps_value + btps_value + ytps_value + upcl_value + jindal_value
# Sharavati Production
sharavati_value = IN.read_value_from_span_id(html, 'lblshvytot')
# Nagjhari Production
nagjhari_value = IN.read_value_from_span_id(html, 'lblngjtot')
# Varahi Production
varahi_value = IN.read_value_from_span_id(html, 'lblvrhtot')
# Kodsalli Production
kodsalli_value = IN.read_value_from_span_id(html, 'lblkdsltot')
# Kadra Production
kadra_value = IN.read_value_from_span_id(html, 'lblkdrtot')
# GERUSOPPA production
gerusoppa_value = IN.read_value_from_span_id(html, 'lblgrsptot')
# JOG production
jog_value = IN.read_value_from_span_id(html, 'lbljogtot')
# LPH Production
lph_value = IN.read_value_from_span_id(html, 'lbllphtot')
# Supa production
supa_value = IN.read_value_from_span_id(html, 'lblsupatot')
# SHIMSHA
shimsha_value = IN.read_value_from_span_id(html, 'lblshimtot')
# SHIVASAMUDRA
shivasamudra_value = IN.read_value_from_span_id(html, 'lblshivtot')
# MANIDAM
manidam_value = IN.read_value_from_span_id(html, 'lblmanitot')
# MUNRABAD
munrabad_value = IN.read_value_from_span_id(html, 'lblmbdtot')
# BHADRA
bhadra_value = IN.read_value_from_span_id(html, 'lblbdratot')
# GHATAPRABHA
ghataprabha_value = IN.read_value_from_span_id(html, 'lblgtprtot')
# ALMATTI
almatti_value = IN.read_value_from_span_id(html, 'lblalmttot')
# CGS Production
# TODO: Search CGS production type
cgs_value = IN.read_value_from_span_id(html, 'lblcgs')
# NCEP Production
ncep_html = web.get_response_soup(country_code, 'http://kptclsldc.com/StateNCEP.aspx', session)
ncep_date_time = IN.read_datetime_from_span_id(ncep_html, 'Label1', 'DD/MM/YYYY HH:mm:ss')
# Check ncep date is similar than state gen date
if abs(india_date_time.timestamp - ncep_date_time.timestamp) > 600:
raise ParserException('IN-KA', 'NCEP or State datetime is not valid')
biomass_value = IN.read_value_from_span_id(ncep_html, 'lbl_tb')
# TODO: Cogeneration value production type?
cogen_value = IN.read_value_from_span_id(ncep_html, 'lbl_tc')
mini_hydro_value = IN.read_value_from_span_id(ncep_html, 'lbl_tm')
wind_value = IN.read_value_from_span_id(ncep_html, 'lbl_tw')
solar_value = IN.read_value_from_span_id(ncep_html, 'lbl_ts')
# Hydro production
hydro_value = sharavati_value + nagjhari_value + varahi_value + kodsalli_value \
+ kadra_value + gerusoppa_value + jog_value + lph_value + supa_value \
+ shimsha_value + shivasamudra_value + manidam_value + munrabad_value \
+ bhadra_value + ghataprabha_value + almatti_value + mini_hydro_value
# Unknown production
unknown_value = cgs_value + cogen_value
data = {
'countryCode': country_code,
'datetime': india_date_time.datetime,
'production': {
'biomass': biomass_value,
'coal': coal_value,
'gas': 0.0,
'hydro': hydro_value,
'nuclear': 0.0,
'oil': 0.0,
'solar': solar_value,
'wind': wind_value,
'geothermal': 0.0,
'unknown': unknown_value
},
'storage': {
'hydro': 0.0
},
'source': 'kptclsldc.com',
}
return data
def fetch_data(country_code, session=None):
countrycode.assert_country_code(country_code, 'IN-GJ')
solar_html = web.get_response_soup(
country_code, 'https://www.sldcguj.com/RealTimeData/GujSolar.php',
session)
wind_html = web.get_response_soup(
country_code, 'https://www.sldcguj.com/RealTimeData/wind.php', session)
india_date = get(
solar_html.find_all('tr')[0].text.split('\t')[-1].strip() +
' Asia/Kolkata', 'DD-MM-YYYY H:m:s ZZZ')
solar_value = float(
literal_eval(
solar_html.find_all('tr')[-1].find_all('td')[-1].text.strip()))
wind_value = float(
literal_eval(
wind_html.find_all('tr')[-1].find_all('td')[-1].text.strip()))
hydro_value = thermal_value = gas_value = coal_value = 0.0
value_map = {
"date": india_date.datetime,
"solar": solar_value,
"hydro": hydro_value,
"thermal": thermal_value,
"wind": wind_value,
"gas": gas_value,
"coal": coal_value
}
cookies_params = {
'ASPSESSIONIDSUQQQTRD': 'ODMNNHADJFGCMLFFGFEMOGBL',
'PHPSESSID': 'a301jk6p1p8d50dduflceeg6l1'
}
# other_html = requests.get('https://www.sldcguj.com/RealTimeData/RealTimeDemand.php', params=cookies_params)
rows = web.get_response_soup(
country_code,
'https://www.sldcguj.com/RealTimeData/RealTimeDemand.php',
session).find_all('tr')
for row in rows:
if len(row.find_all('td')) > 3:
v1, v2 = (re.sub(r'[\n\t\r]', r'', x.text.strip())
for x in itemgetter(*[0, 3])(row.find_all('td')))
energy_type = [k for k, v in station_map.items() if v1 in v]
if len(energy_type) > 0:
value_map[energy_type[0]] += float(literal_eval(v2))
elif len(row.find_all('td')) == 3:
v1, v2 = (re.sub(r'[\n\t\r]', r'', x.text.strip())
for x in itemgetter(*[0, 2])(row.find_all('td')))
energy_type = [k for k, v in station_map.items() if v1 in v]
if len(energy_type) > 0:
value_map[energy_type[0]] += float(literal_eval(v2))
if v1 == 'Gujarat Catered':
value_map['total consumption'] = float(
literal_eval(v2.split(' ')[0]))
return value_map
def fetch_production(country_code='IN-KA', session=None):
"""Fetch Karnataka production"""
countrycode.assert_country_code(country_code, 'IN-KA')
html = web.get_response_soup(country_code,
'http://kptclsldc.com/StateGen.aspx', session)
india_date_time = IN.read_datetime_from_span_id(html, 'lbldate',
'M/D/YYYY h:mm:ss A')
# RTPS Production: https://en.wikipedia.org/wiki/Raichur_Thermal_Power_Station
rtps_value = IN.read_value_from_span_id(html, 'lblrtptot')
# BTPS Production: https://en.wikipedia.org/wiki/Bellary_Thermal_Power_station
btps_value = IN.read_value_from_span_id(html, 'lblbtptot')
# YTPS Production: https://en.wikipedia.org/wiki/Yermarus_Thermal_Power_Station
ytps_value = IN.read_value_from_span_id(html, 'ytptot')
# UPCL Production: https://en.wikipedia.org/wiki/Udupi_Power_Plant
upcl_value = IN.read_value_from_span_id(html, 'lblupctot')
# JINDAl Production: https://en.wikipedia.org/wiki/JSW_Vijayanagar_Power_Station
jindal_value = IN.read_value_from_span_id(html, 'lbljintot')
# Coal Production
coal_value = rtps_value + btps_value + ytps_value + upcl_value + jindal_value
# Sharavati Production: Sharavati Hydroelectric
sharavati_value = IN.read_value_from_span_id(html, 'lblshvytot')
# Nagjhari Production: Kalinadi-Nagjhari Hydroelectric
nagjhari_value = IN.read_value_from_span_id(html, 'lblngjtot')
# Varahi Production: https://en.wikipedia.org/wiki/Varahi_River#Varahi_Hydro-electric_Project
varahi_value = IN.read_value_from_span_id(html, 'lblvrhtot')
# Kodsalli Production: Kalinadi Kodasalli Hydroelectric
kodsalli_value = IN.read_value_from_span_id(html, 'lblkdsltot')
# Kadra Production: https://en.wikipedia.org/wiki/Kadra_Dam
kadra_value = IN.read_value_from_span_id(html, 'lblkdrtot')
# GERUSOPPA production: Gerusoppa Dam
gerusoppa_value = IN.read_value_from_span_id(html, 'lblgrsptot')
# JOG production: https://en.wikipedia.org/wiki/Jog_Falls
jog_value = IN.read_value_from_span_id(html, 'lbljogtot')
# LPH Production: Linganamakki Dam
lph_value = IN.read_value_from_span_id(html, 'lbllphtot')
# Supa production: https://en.wikipedia.org/wiki/Supa_Dam
supa_value = IN.read_value_from_span_id(html, 'lblsupatot')
# SHIMSHA: https://en.wikipedia.org/wiki/Shimsha#Power_generation
shimsha_value = IN.read_value_from_span_id(html, 'lblshimtot')
# SHIVASAMUDRA: https://en.wikipedia.org/wiki/Shivanasamudra_Falls#Power_generation
shivasamudra_value = IN.read_value_from_span_id(html, 'lblshivtot')
# MANIDAM: Mani Dam Hydroelectric
manidam_value = IN.read_value_from_span_id(html, 'lblmanitot')
# MUNRABAD: Munirabad Hydroelectric
munrabad_value = IN.read_value_from_span_id(html, 'lblmbdtot')
# BHADRA: https://en.wikipedia.org/wiki/Bhadra_Dam
bhadra_value = IN.read_value_from_span_id(html, 'lblbdratot')
# GHATAPRABHA: Ghataprabha Hydroelectric
ghataprabha_value = IN.read_value_from_span_id(html, 'lblgtprtot')
# ALMATTI: https://en.wikipedia.org/wiki/Almatti_Dam
almatti_value = IN.read_value_from_span_id(html, 'lblalmttot')
# CGS (Central Generating Stations) Production
# TODO: Search CGS production type
cgs_value = IN.read_value_from_span_id(html, 'lblcgs')
# NCEP (Non-Conventional Energy Production)
ncep_html = web.get_response_soup(country_code,
'http://kptclsldc.com/StateNCEP.aspx',
session)
ncep_date_time = IN.read_datetime_from_span_id(ncep_html, 'Label1',
'DD/MM/YYYY HH:mm:ss')
# Check ncep date is similar than state gen date
if abs(india_date_time.timestamp - ncep_date_time.timestamp) > 600:
raise ParserException('IN-KA', 'NCEP or State datetime is not valid')
biomass_value = IN.read_value_from_span_id(ncep_html, 'lbl_tb')
# TODO: Cogeneration value production type?
cogen_value = IN.read_value_from_span_id(ncep_html, 'lbl_tc')
mini_hydro_value = IN.read_value_from_span_id(ncep_html, 'lbl_tm')
wind_value = IN.read_value_from_span_id(ncep_html, 'lbl_tw')
solar_value = IN.read_value_from_span_id(ncep_html, 'lbl_ts')
# Hydro production
hydro_value = sharavati_value + nagjhari_value + varahi_value + kodsalli_value \
+ kadra_value + gerusoppa_value + jog_value + lph_value + supa_value \
+ shimsha_value + shivasamudra_value + manidam_value + munrabad_value \
+ bhadra_value + ghataprabha_value + almatti_value + mini_hydro_value
# Unknown production
unknown_value = cgs_value + cogen_value
data = {
'countryCode': country_code,
'datetime': india_date_time.datetime,
'production': {
'biomass': biomass_value,
'coal': coal_value,
'gas': 0.0,
'hydro': hydro_value,
'nuclear': 0.0,
'oil': 0.0,
'solar': solar_value,
'wind': wind_value,
'geothermal': 0.0,
'unknown': unknown_value
},
'storage': {
'hydro': 0.0
},
'source': 'kptclsldc.com',
}
return data
