def test_get_response_soup(self): try: response_soup = web.get_response_soup('ES', 'https://www.google.es') self.assertIsNotNone(response_soup) except ParserException as ex: self.fail("assert_country_code() raised Exception unexpectedly!")
def test_get_response_soup(self): try: response_soup = web.get_response_soup("ES", "https://www.google.es") self.assertIsNotNone(response_soup) except ParserException as ex: self.fail("assert_zone_key() raised Exception unexpectedly!")
def fetch_production(country_code='IN-CT', session=None): """Fetch Chhattisgarh production""" countrycode.assert_country_code(country_code, 'IN-CT') html = web.get_response_soup(country_code, 'http://117.239.199.203/csptcl/GEN.aspx', session) india_date_time = IN.read_datetime_from_span_id(html, 'L37', 'hh:m DD-MM-YY') korba_east_value = IN.read_value_from_span_id(html, 'L7') korba_west_value = IN.read_value_from_span_id(html, 'L13') dsmp_value = IN.read_value_from_span_id(html, 'L16') marwa_value = IN.read_value_from_span_id(html, 'L23') coal_value = round(korba_east_value + korba_west_value + dsmp_value + marwa_value, 2) bango_value = IN.read_value_from_span_id(html, 'L20') data = { 'countryCode': country_code, 'datetime': india_date_time.datetime, 'production': { 'coal': coal_value, 'hydro': bango_value }, 'source': 'cspc.co.in', } return data
def fetch_production(country_code='IN-DL', session=None): """Fetch Delhi production""" countrycode.assert_country_code(country_code, 'IN-DL') html = web.get_response_soup( country_code, 'http://www.delhisldc.org/Redirect.aspx?Loc=0804', session) india_date_string = IN.read_text_from_span_id( html, 'ContentPlaceHolder3_ddgenco') india_date_time = IN.read_datetime_with_only_time(india_date_string, 'HH:mm:ss') prod_table = html.find("table", {"id": "ContentPlaceHolder3_dgenco"}) prod_rows = prod_table.findAll('tr') # BTPS https://en.wikipedia.org/wiki/Badarpur_Thermal_Power_Station btps = read_value(prod_rows[1]) # CCGT https://en.wikipedia.org/wiki/Pragati-III_Combined_Cycle_Power_Plant = Pragati-3 ccgt = read_value(prod_rows[2]) # DMSWSL (Delhi Municipal Solid Waste Solutions Limited): Garbage-to-electricity dmswsl = read_value(prod_rows[3]) # EDWPL (East Delhi Waste Processing Company Limited): Garbage-to-electricity edwpl = read_value(prod_rows[4]) # GT (Gas Turbine) https://en.wikipedia.org/wiki/IPGCL_Gas_Turbine_Power_Station gt = read_value(prod_rows[5]) # Pragati https://en.wikipedia.org/wiki/Pragati-I_Combined_Cycle_Gas_Power_Station = Pragati-1 pragati = read_value(prod_rows[6]) # TOWMP (Timarpur Okhla Waste Management Company Pvt. Ltd.): Garbage-to-electricity towmp = read_value(prod_rows[7]) # Coal production coal = btps # Gas production gas = ccgt + pragati + gt # Unknown production garbage = dmswsl + edwpl + towmp data = { 'countryCode': country_code, 'datetime': india_date_time.datetime, 'production': { 'coal': coal, 'gas': gas, 'biomass': garbage }, 'source': 'delhisldc.org', } return data
def fetch_production(country_code='IN-DL', session=None): """Fetch Delhi production""" countrycode.assert_country_code(country_code, 'IN-DL') html = web.get_response_soup( country_code, 'http://www.delhisldc.org/Redirect.aspx?Loc=0804', session) india_date_string = IN.read_text_from_span_id( html, 'ContentPlaceHolder3_ddgenco') india_date_time = IN.read_datetime_with_only_time(india_date_string, 'HH:mm:ss') prod_table = html.find("table", {"id": "ContentPlaceHolder3_dgenco"}) prod_rows = prod_table.findAll('tr') # BTPS https://en.wikipedia.org/wiki/Badarpur_Thermal_Power_Station btps = read_value(prod_rows[1]) # CCGT https://en.wikipedia.org/wiki/Pragati-III_Combined_Cycle_Power_Plant ccgt = read_value(prod_rows[2]) # DMSWSL Unknown dmswsl = read_value(prod_rows[3]) # EDWPL Unknown edwpl = read_value(prod_rows[4]) # GT Unknown gt = read_value(prod_rows[5]) # Pragati pragati = read_value(prod_rows[6]) # TOWMP Waste? towmp = read_value(prod_rows[7]) # Coal production coal = btps # Gas production gas = ccgt + pragati # Unknown production unknown_value = dmswsl + edwpl + gt + towmp data = { 'countryCode': country_code, 'datetime': india_date_time.datetime, 'production': { 'coal': coal, 'gas': gas, 'unknown': unknown_value }, 'source': 'delhisldc.org', } return data
def fetch_production(country_code='IN-DL', session=None): """Fetch Delhi production""" countrycode.assert_country_code(country_code, 'IN-DL') html = web.get_response_soup(country_code, 'http://www.delhisldc.org/Redirect.aspx?Loc=0804', session) india_date_string = IN.read_text_from_span_id(html, 'ContentPlaceHolder3_ddgenco') india_date_time = IN.read_datetime_with_only_time(india_date_string, 'HH:mm:ss') prod_table = html.find("table", {"id": "ContentPlaceHolder3_dgenco"}) prod_rows = prod_table.findAll('tr') # BTPS https://en.wikipedia.org/wiki/Badarpur_Thermal_Power_Station btps = read_value(prod_rows[1]) # CCGT https://en.wikipedia.org/wiki/Pragati-III_Combined_Cycle_Power_Plant ccgt = read_value(prod_rows[2]) # DMSWSL Unknown dmswsl = read_value(prod_rows[3]) # EDWPL Unknown edwpl = read_value(prod_rows[4]) # GT Unknown gt = read_value(prod_rows[5]) # Pragati pragati = read_value(prod_rows[6]) # TOWMP Waste? towmp = read_value(prod_rows[7]) # Coal production coal = btps # Gas production gas = ccgt + pragati # Unknown production unknown_value = dmswsl + edwpl + gt + towmp data = { 'countryCode': country_code, 'datetime': india_date_time.datetime, 'production': { 'coal': coal, 'gas': gas, 'unknown': unknown_value }, 'source': 'delhisldc.org', } return data
def fetch_consumption(country_code='IN-DL', session=None): """Fetch Delhi consumption""" countrycode.assert_country_code(country_code, 'IN-DL') html = web.get_response_soup(country_code, 'http://www.delhisldc.org/Redirect.aspx', session) india_date_time = IN.read_datetime_from_span_id(html, 'DynamicData1_LblDate', 'DD-MMM-YYYY hh:mm:ss A') demand_value = IN.read_value_from_span_id(html, 'DynamicData1_LblLoad') data = { 'countryCode': country_code, 'datetime': india_date_time.datetime, 'consumption': demand_value, 'source': 'delhisldc.org' } return data
def fetch_consumption(country_code='IN-CT', session=None): """Fetch Chhattisgarh consumption""" countrycode.assert_country_code(country_code, 'IN-CT') html = web.get_response_soup(country_code, 'http://117.239.199.203/csptcl/GEN.aspx', session) india_date_time = IN.read_datetime_from_span_id(html, 'L37', 'hh:m DD-MM-YY') demand_value = IN.read_value_from_span_id(html, 'L29') data = { 'countryCode': country_code, 'datetime': india_date_time.datetime, 'consumption': demand_value, 'source': 'cspc.co.in' } return data
def fetch_consumption(country_code='IN-KA', session=None): """Fetch Karnataka consumption""" countrycode.assert_country_code(country_code, 'IN-KA') html = web.get_response_soup(country_code, 'http://kptclsldc.com/Default.aspx', session) india_date_time = IN.read_datetime_from_span_id(html, 'Label6', 'DD/MM/YYYY HH:mm') demand_value = IN.read_value_from_span_id(html, 'Label5') data = { 'countryCode': country_code, 'datetime': india_date_time.datetime, 'consumption': demand_value, 'source': 'kptclsldc.com' } return data
def fetch_consumption(country_code='IN-KA', session=None): """Fetch Karnataka consumption""" countrycode.assert_country_code(country_code, 'IN-KA') html = web.get_response_soup(country_code, 'http://kptclsldc.com/Default.aspx', session) india_date_time = india.read_datetime_from_span_id(html, 'Label6', 'DD/MM/YYYY HH:mm') demand_value = india.read_value_from_span_id(html, 'Label5') data = { 'countryCode': country_code, 'datetime': india_date_time.datetime, 'consumption': demand_value, 'source': 'kptclsldc.com' } return data
def fetch_production(country_code='IN-KA', session=None): """Fetch Karnataka production""" countrycode.assert_country_code(country_code, 'IN-KA') html = web.get_response_soup(country_code, 'http://kptclsldc.com/StateGen.aspx', session) india_date_time = IN.read_datetime_from_span_id(html, 'lbldate', 'M/D/YYYY h:mm:ss A') # RTPS Production: https://en.wikipedia.org/wiki/Raichur_Thermal_Power_Station rtps_value = IN.read_value_from_span_id(html, 'lblrtptot') # BTPS Production: https://en.wikipedia.org/wiki/Bellary_Thermal_Power_station btps_value = IN.read_value_from_span_id(html, 'lblbtptot') # YTPS Production: https://en.wikipedia.org/wiki/Yermarus_Thermal_Power_Station ytps_value = IN.read_value_from_span_id(html, 'ytptot') # UPCL Production: https://en.wikipedia.org/wiki/Udupi_Power_Plant upcl_value = IN.read_value_from_span_id(html, 'lblupctot') # JINDAl Production jindal_value = IN.read_value_from_span_id(html, 'lbljintot') # Coal Production coal_value = rtps_value + btps_value + ytps_value + upcl_value + jindal_value # Sharavati Production sharavati_value = IN.read_value_from_span_id(html, 'lblshvytot') # Nagjhari Production nagjhari_value = IN.read_value_from_span_id(html, 'lblngjtot') # Varahi Production varahi_value = IN.read_value_from_span_id(html, 'lblvrhtot') # Kodsalli Production kodsalli_value = IN.read_value_from_span_id(html, 'lblkdsltot') # Kadra Production kadra_value = IN.read_value_from_span_id(html, 'lblkdrtot') # GERUSOPPA production gerusoppa_value = IN.read_value_from_span_id(html, 'lblgrsptot') # JOG production jog_value = IN.read_value_from_span_id(html, 'lbljogtot') # LPH Production lph_value = IN.read_value_from_span_id(html, 'lbllphtot') # Supa production supa_value = IN.read_value_from_span_id(html, 'lblsupatot') # SHIMSHA shimsha_value = IN.read_value_from_span_id(html, 'lblshimtot') # SHIVASAMUDRA shivasamudra_value = IN.read_value_from_span_id(html, 'lblshivtot') # MANIDAM manidam_value = IN.read_value_from_span_id(html, 'lblmanitot') # MUNRABAD munrabad_value = IN.read_value_from_span_id(html, 'lblmbdtot') # BHADRA bhadra_value = IN.read_value_from_span_id(html, 'lblbdratot') # GHATAPRABHA ghataprabha_value = IN.read_value_from_span_id(html, 'lblgtprtot') # ALMATTI almatti_value = IN.read_value_from_span_id(html, 'lblalmttot') # CGS Production # TODO: Search CGS production type cgs_value = IN.read_value_from_span_id(html, 'lblcgs') # NCEP Production ncep_html = web.get_response_soup(country_code, 'http://kptclsldc.com/StateNCEP.aspx', session) ncep_date_time = IN.read_datetime_from_span_id(ncep_html, 'Label1', 'DD/MM/YYYY HH:mm:ss') # Check ncep date is similar than state gen date if abs(india_date_time.timestamp - ncep_date_time.timestamp) > 600: raise ParserException('IN-KA', 'NCEP or State datetime is not valid') biomass_value = IN.read_value_from_span_id(ncep_html, 'lbl_tb') # TODO: Cogeneration value production type? cogen_value = IN.read_value_from_span_id(ncep_html, 'lbl_tc') mini_hydro_value = IN.read_value_from_span_id(ncep_html, 'lbl_tm') wind_value = IN.read_value_from_span_id(ncep_html, 'lbl_tw') solar_value = IN.read_value_from_span_id(ncep_html, 'lbl_ts') # Hydro production hydro_value = sharavati_value + nagjhari_value + varahi_value + kodsalli_value \ + kadra_value + gerusoppa_value + jog_value + lph_value + supa_value \ + shimsha_value + shivasamudra_value + manidam_value + munrabad_value \ + bhadra_value + ghataprabha_value + almatti_value + mini_hydro_value # Unknown production unknown_value = cgs_value + cogen_value data = { 'countryCode': country_code, 'datetime': india_date_time.datetime, 'production': { 'biomass': biomass_value, 'coal': coal_value, 'gas': 0.0, 'hydro': hydro_value, 'nuclear': 0.0, 'oil': 0.0, 'solar': solar_value, 'wind': wind_value, 'geothermal': 0.0, 'unknown': unknown_value }, 'storage': { 'hydro': 0.0 }, 'source': 'kptclsldc.com', } return data
def fetch_data(country_code, session=None): countrycode.assert_country_code(country_code, 'IN-GJ') solar_html = web.get_response_soup( country_code, 'https://www.sldcguj.com/RealTimeData/GujSolar.php', session) wind_html = web.get_response_soup( country_code, 'https://www.sldcguj.com/RealTimeData/wind.php', session) india_date = get( solar_html.find_all('tr')[0].text.split('\t')[-1].strip() + ' Asia/Kolkata', 'DD-MM-YYYY H:m:s ZZZ') solar_value = float( literal_eval( solar_html.find_all('tr')[-1].find_all('td')[-1].text.strip())) wind_value = float( literal_eval( wind_html.find_all('tr')[-1].find_all('td')[-1].text.strip())) hydro_value = thermal_value = gas_value = coal_value = 0.0 value_map = { "date": india_date.datetime, "solar": solar_value, "hydro": hydro_value, "thermal": thermal_value, "wind": wind_value, "gas": gas_value, "coal": coal_value } cookies_params = { 'ASPSESSIONIDSUQQQTRD': 'ODMNNHADJFGCMLFFGFEMOGBL', 'PHPSESSID': 'a301jk6p1p8d50dduflceeg6l1' } # other_html = requests.get('https://www.sldcguj.com/RealTimeData/RealTimeDemand.php', params=cookies_params) rows = web.get_response_soup( country_code, 'https://www.sldcguj.com/RealTimeData/RealTimeDemand.php', session).find_all('tr') for row in rows: if len(row.find_all('td')) > 3: v1, v2 = (re.sub(r'[\n\t\r]', r'', x.text.strip()) for x in itemgetter(*[0, 3])(row.find_all('td'))) energy_type = [k for k, v in station_map.items() if v1 in v] if len(energy_type) > 0: value_map[energy_type[0]] += float(literal_eval(v2)) elif len(row.find_all('td')) == 3: v1, v2 = (re.sub(r'[\n\t\r]', r'', x.text.strip()) for x in itemgetter(*[0, 2])(row.find_all('td'))) energy_type = [k for k, v in station_map.items() if v1 in v] if len(energy_type) > 0: value_map[energy_type[0]] += float(literal_eval(v2)) if v1 == 'Gujarat Catered': value_map['total consumption'] = float( literal_eval(v2.split(' ')[0])) return value_map
def fetch_production(country_code='IN-KA', session=None): """Fetch Karnataka production""" countrycode.assert_country_code(country_code, 'IN-KA') html = web.get_response_soup(country_code, 'http://kptclsldc.com/StateGen.aspx', session) india_date_time = IN.read_datetime_from_span_id(html, 'lbldate', 'M/D/YYYY h:mm:ss A') # RTPS Production: https://en.wikipedia.org/wiki/Raichur_Thermal_Power_Station rtps_value = IN.read_value_from_span_id(html, 'lblrtptot') # BTPS Production: https://en.wikipedia.org/wiki/Bellary_Thermal_Power_station btps_value = IN.read_value_from_span_id(html, 'lblbtptot') # YTPS Production: https://en.wikipedia.org/wiki/Yermarus_Thermal_Power_Station ytps_value = IN.read_value_from_span_id(html, 'ytptot') # UPCL Production: https://en.wikipedia.org/wiki/Udupi_Power_Plant upcl_value = IN.read_value_from_span_id(html, 'lblupctot') # JINDAl Production: https://en.wikipedia.org/wiki/JSW_Vijayanagar_Power_Station jindal_value = IN.read_value_from_span_id(html, 'lbljintot') # Coal Production coal_value = rtps_value + btps_value + ytps_value + upcl_value + jindal_value # Sharavati Production: Sharavati Hydroelectric sharavati_value = IN.read_value_from_span_id(html, 'lblshvytot') # Nagjhari Production: Kalinadi-Nagjhari Hydroelectric nagjhari_value = IN.read_value_from_span_id(html, 'lblngjtot') # Varahi Production: https://en.wikipedia.org/wiki/Varahi_River#Varahi_Hydro-electric_Project varahi_value = IN.read_value_from_span_id(html, 'lblvrhtot') # Kodsalli Production: Kalinadi Kodasalli Hydroelectric kodsalli_value = IN.read_value_from_span_id(html, 'lblkdsltot') # Kadra Production: https://en.wikipedia.org/wiki/Kadra_Dam kadra_value = IN.read_value_from_span_id(html, 'lblkdrtot') # GERUSOPPA production: Gerusoppa Dam gerusoppa_value = IN.read_value_from_span_id(html, 'lblgrsptot') # JOG production: https://en.wikipedia.org/wiki/Jog_Falls jog_value = IN.read_value_from_span_id(html, 'lbljogtot') # LPH Production: Linganamakki Dam lph_value = IN.read_value_from_span_id(html, 'lbllphtot') # Supa production: https://en.wikipedia.org/wiki/Supa_Dam supa_value = IN.read_value_from_span_id(html, 'lblsupatot') # SHIMSHA: https://en.wikipedia.org/wiki/Shimsha#Power_generation shimsha_value = IN.read_value_from_span_id(html, 'lblshimtot') # SHIVASAMUDRA: https://en.wikipedia.org/wiki/Shivanasamudra_Falls#Power_generation shivasamudra_value = IN.read_value_from_span_id(html, 'lblshivtot') # MANIDAM: Mani Dam Hydroelectric manidam_value = IN.read_value_from_span_id(html, 'lblmanitot') # MUNRABAD: Munirabad Hydroelectric munrabad_value = IN.read_value_from_span_id(html, 'lblmbdtot') # BHADRA: https://en.wikipedia.org/wiki/Bhadra_Dam bhadra_value = IN.read_value_from_span_id(html, 'lblbdratot') # GHATAPRABHA: Ghataprabha Hydroelectric ghataprabha_value = IN.read_value_from_span_id(html, 'lblgtprtot') # ALMATTI: https://en.wikipedia.org/wiki/Almatti_Dam almatti_value = IN.read_value_from_span_id(html, 'lblalmttot') # CGS (Central Generating Stations) Production # TODO: Search CGS production type cgs_value = IN.read_value_from_span_id(html, 'lblcgs') # NCEP (Non-Conventional Energy Production) ncep_html = web.get_response_soup(country_code, 'http://kptclsldc.com/StateNCEP.aspx', session) ncep_date_time = IN.read_datetime_from_span_id(ncep_html, 'Label1', 'DD/MM/YYYY HH:mm:ss') # Check ncep date is similar than state gen date if abs(india_date_time.timestamp - ncep_date_time.timestamp) > 600: raise ParserException('IN-KA', 'NCEP or State datetime is not valid') biomass_value = IN.read_value_from_span_id(ncep_html, 'lbl_tb') # TODO: Cogeneration value production type? cogen_value = IN.read_value_from_span_id(ncep_html, 'lbl_tc') mini_hydro_value = IN.read_value_from_span_id(ncep_html, 'lbl_tm') wind_value = IN.read_value_from_span_id(ncep_html, 'lbl_tw') solar_value = IN.read_value_from_span_id(ncep_html, 'lbl_ts') # Hydro production hydro_value = sharavati_value + nagjhari_value + varahi_value + kodsalli_value \ + kadra_value + gerusoppa_value + jog_value + lph_value + supa_value \ + shimsha_value + shivasamudra_value + manidam_value + munrabad_value \ + bhadra_value + ghataprabha_value + almatti_value + mini_hydro_value # Unknown production unknown_value = cgs_value + cogen_value data = { 'countryCode': country_code, 'datetime': india_date_time.datetime, 'production': { 'biomass': biomass_value, 'coal': coal_value, 'gas': 0.0, 'hydro': hydro_value, 'nuclear': 0.0, 'oil': 0.0, 'solar': solar_value, 'wind': wind_value, 'geothermal': 0.0, 'unknown': unknown_value }, 'storage': { 'hydro': 0.0 }, 'source': 'kptclsldc.com', } return data