def get(self):
""" Attempt to get the initial version of the form from the website. """
if self.cookies is None:
get_or_post_a_url(_make_url('Default.aspx', False))
response = get_or_post_a_url(_make_url('ReportManager.aspx?ReportVisibility=1&ReportCategory=0'))
self.cookies = {'ASP.NET_SessionId': response.cookies.get('ASP.NET_SessionId')}
response = get_or_post_a_url(self.start_url, cookies=self.cookies)
self.form_data = FormData(response.content)
if self.action_url is None:
self.action_url = _make_url(self.form_data.action)
return True
def _do_post(self, submit=False):
""" Submit the form data and update based on response.
Given how slow the parsing of a 3M HTML page is, try and use the
X-MicrosoftAjax: Delta=true header to get smaller blocks for processing.
"""
if self.form_data.action is None:
self.logger.info("Unable to post due no action URL available. Did you call get()?")
return None
action_url = _make_url(self.form_data.action)
form_hdrs = {'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'User-Agent': 'Mozilla',
'X-Requested-With': 'XMLHttpRequest',
'X-MicrosoftAjax': 'Delta=true',
'Referer': unquote(action_url)
}
post_dict = self.form_data.as_post_data(submit=submit)
post_data = "&".join(["{}={}".format(key, post_dict[key]) for key in post_dict.keys()])
response = get_or_post_a_url(action_url,
post=True,
cookies=self.cookies,
headers=form_hdrs,
data=post_data)
return response
def get(self):
""" Attempt to get the initial version of the form from the website. """
if self.cookies is None:
get_or_post_a_url(_make_url('Default.aspx', False))
response = get_or_post_a_url(
_make_url(
'ReportManager.aspx?ReportVisibility=1&ReportCategory=0'))
self.cookies = {
'ASP.NET_SessionId': response.cookies.get('ASP.NET_SessionId')
}
response = get_or_post_a_url(self.start_url, cookies=self.cookies)
self.form_data = FormData(response.content)
if self.action_url is None:
self.action_url = _make_url(self.form_data.action)
return True
def _do_post(self, submit=False):
""" Submit the form data and update based on response.
Given how slow the parsing of a 3M HTML page is, try and use the
X-MicrosoftAjax: Delta=true header to get smaller blocks for processing.
"""
if self.form_data.action is None:
self.logger.info(
"Unable to post due no action URL available. Did you call get()?"
)
return None
action_url = _make_url(self.form_data.action)
form_hdrs = {
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'User-Agent': 'Mozilla',
'X-Requested-With': 'XMLHttpRequest',
'X-MicrosoftAjax': 'Delta=true',
'Referer': unquote(action_url)
}
post_dict = self.form_data.as_post_data(submit=submit)
post_data = "&".join(
["{}={}".format(key, post_dict[key]) for key in post_dict.keys()])
response = get_or_post_a_url(action_url,
post=True,
cookies=self.cookies,
headers=form_hdrs,
data=post_data)
return response
def get_data(self):
""" Get the data from the DECC server and parse it into DECC records.
:returns: True or False
:rtype: bool
"""
if self.filename is not None:
return self._parse_filename()
if self.available is None:
self._find_available()
if self.available is None:
raise Exception("Unable to get details of available downloads")
response = get_or_post_a_url(self.available['url'])
self.raw_data = response.content
if sys.version_info >= (3, 0):
csvfile = csv.reader(
codecs.iterdecode(response.content.splitlines(), 'latin1'))
else:
csvfile = csv.reader(response.content.splitlines())
for row in csvfile:
self._parse_row(row)
self.records = sorted(self.records, key=lambda rec: rec.site_name)
return True
def get_data(self):
""" Get the data from the DECC server and parse it into DECC records.
:returns: True or False
:rtype: bool
"""
if self.filename is not None:
return self._parse_filename()
if self.available is None:
self._find_available()
if self.available is None:
raise Exception("Unable to get details of available downloads")
response = get_or_post_a_url(self.available['url'])
self.raw_data = response.content
if sys.version_info >= (3, 0):
csvfile = csv.reader(codecs.iterdecode(response.content.splitlines(), 'latin1'))
else:
csvfile = csv.reader(response.content.splitlines())
for row in csvfile:
self._parse_row(row)
self.records = sorted(self.records, key=lambda rec: rec.site_name)
return True
def get_data(self, **params):
""" Get data from the Elexon servers and attempt to parse it into a series of
dicts each representing a record. Parameters are passed as a dict.
Multiple sets of data are created in the multi member, single sets in items.
"""
if self.report is None:
raise Exception(
"ElexonAPI objects require the report be set before use.")
if self.apikey is None:
raise Exception(
"An API key is required to use the Elexon API accessor functionality"
)
url = make_elexon_url(self.report, self.version)
params.update({'APIKey': self.apikey, 'ServiceType': 'xml'})
req = get_or_post_a_url(url, params=params)
# print(req.content)
xml = parse_response_as_xml(req)
http = xml.xpath('/response/responseMetadata/httpCode')
response_code = int(http[0].text)
if response_code == 204:
print("No content returned, but no error reported.")
return True
elif response_code != 200:
print("No data returned. Error reported.")
err = xml.xpath('/response/responseMetadata/description')
print(err[0].text)
return False
if self.MULTI_RESULTS is None:
for item in xml.xpath('/response/responseBody/responseList/item'):
item_dict = map_xml_to_dict(item, self.XML_MAPPING)
if 'activeflag' in item_dict:
item_dict['activeflag'] = item_dict['activeflag'] == 'Y'
if 'settlementperiod' in item_dict:
item_dict['settlementperiod'] = int(
item_dict['settlementperiod'])
self.post_item_cleanup(item_dict)
self.items.append(item_dict)
else:
for result_set in self.MULTI_RESULTS:
self.multi[result_set[0]] = []
for item in xml.xpath(result_set[1]):
item_dict = map_xml_to_dict(item, self.XML_MAPPING)
if 'activeFlags' in item_dict:
item_dict['activeflag'] = item_dict[
'activeflag'] == 'Y'
if 'settlementperiod' in item_dict:
item_dict['settlementperiod'] = int(
item_dict['settlementperiod'])
self.post_item_cleanup(item_dict)
# print(item_dict)
self.multi[result_set[0]].append(item_dict)
return True
def get_data(self):
""" Get data from the BM Reports website. Try 3 times.
"""
resp = get_or_post_a_url(self.URL, params=self.PARAMS)
self.xml = parse_response_as_xml(resp)
if self.xml is None:
return
for section in ['INST', 'HH', 'LAST24H']:
self.sections.append(GenerationPeriod(self.xml.xpath(section)[0]))
def get_data(self):
""" Get data from the BM Reports website. Try 3 times.
"""
resp = get_or_post_a_url(self.URL, params=self.PARAMS)
self.xml = parse_response_as_xml(resp)
if self.xml is None:
return
for section in ['INST', 'HH', 'LAST24H']:
self.sections.append(GenerationPeriod(self.xml.xpath(section)[0]))
def get_prices(self):
"""
Get the recent prices.
:return: True or False
:rtype: bool
"""
resp = get_or_post_a_url(self.URL)
self._parse_content(resp.content)
return True
def get_prices(self):
"""
Get the recent prices.
:return: True or False
:rtype: bool
"""
resp = get_or_post_a_url(self.URL)
self._parse_content(resp.content)
return True
def get_data(self, **params):
""" Get data from the Elexon servers and attempt to parse it into a series of
dicts each representing a record. Parameters are passed as a dict.
Multiple sets of data are created in the multi member, single sets in items.
"""
if self.report is None:
raise Exception("ElexonAPI objects require the report be set before use.")
if self.apikey is None:
raise Exception("An API key is required to use the Elexon API accessor functionality")
url = make_elexon_url(self.report, self.version)
params.update({'APIKey': self.apikey, 'ServiceType': 'xml'})
req = get_or_post_a_url(url, params=params)
# print(req.content)
xml = parse_response_as_xml(req)
http = xml.xpath('/response/responseMetadata/httpCode')
response_code = int(http[0].text)
if response_code == 204:
print("No content returned, but no error reported.")
return True
elif response_code != 200:
print("No data returned. Error reported.")
err = xml.xpath('/response/responseMetadata/description')
print(err[0].text)
return False
if self.MULTI_RESULTS is None:
for item in xml.xpath('/response/responseBody/responseList/item'):
item_dict = map_xml_to_dict(item, self.XML_MAPPING)
if 'activeflag' in item_dict:
item_dict['activeflag'] = item_dict['activeflag'] == 'Y'
if 'settlementperiod' in item_dict:
item_dict['settlementperiod'] = int(item_dict['settlementperiod'])
self.post_item_cleanup(item_dict)
self.items.append(item_dict)
else:
for result_set in self.MULTI_RESULTS:
self.multi[result_set[0]] = []
for item in xml.xpath(result_set[1]):
item_dict = map_xml_to_dict(item, self.XML_MAPPING)
if 'activeFlags' in item_dict:
item_dict['activeflag'] = item_dict['activeflag'] == 'Y'
if 'settlementperiod' in item_dict:
item_dict['settlementperiod'] = int(item_dict['settlementperiod'])
self.post_item_cleanup(item_dict)
# print(item_dict)
self.multi[result_set[0]].append(item_dict)
return True
def _find_available(self):
"""
Get the URL and period for the currently available download.
"""
response = get_or_post_a_url(self.URL)
document = html5lib.parse(response.content,
treebuilder="lxml",
namespaceHTMLElements=False)
titles = document.xpath('.//h2[@class="title"]')
period = None
for tit in titles:
if len(tit.getchildren()) == 0:
period = tit.text.split(':')[1].strip()
links = document.xpath('.//span[@class="download"]/a')
self.available = {'period': period,
'url': self.BASE_URL + links[0].get('href')}
def _find_available(self):
"""
Get the URL and period for the currently available download.
"""
response = get_or_post_a_url(self.URL)
document = html5lib.parse(response.content,
treebuilder="lxml",
namespaceHTMLElements=False)
titles = document.xpath('.//h2[@class="title"]')
period = None
for tit in titles:
if len(tit.getchildren()) == 0:
period = tit.text.split(':')[1].strip()
links = document.xpath('.//span[@class="download"]/a')
self.available = {
'period': period,
'url': self.BASE_URL + links[0].get('href')
}
def get_data(self):
""" Get the data from the remote server. """
data = {'element': 'SYSPRICE', 'dT': self.dtt.strftime("%Y-%m-%d")}
resp = get_or_post_a_url(self.URL, params=data)
self.xml = parse_response_as_xml(resp)
if self.xml is None:
return False
for elm in self.xml.xpath('.//ELEMENT'):
data = {}
for elm2 in elm.getchildren():
if elm2.tag == 'SP':
data['period'] = int(elm2.text)
elif elm2.tag == 'SD':
data['date'] = datetime.strptime(elm2.text, "%Y-%m-%d")
else:
data[elm2.tag.lower()] = elm2.text
self.prices.append(data)
return len(self.prices) > 0
def get_data(self):
""" Get the data from the remote server. """
data = {'element': 'SYSPRICE',
'dT': self.dtt.strftime("%Y-%m-%d")}
resp = get_or_post_a_url(self.URL, params=data)
self.xml = parse_response_as_xml(resp)
if self.xml is None:
return False
for elm in self.xml.xpath('.//ELEMENT'):
data = {}
for elm2 in elm.getchildren():
if elm2.tag == 'SP':
data['period'] = int(elm2.text)
elif elm2.tag == 'SD':
data['date'] = datetime.strptime(elm2.text, "%Y-%m-%d")
else:
data[elm2.tag.lower()] = elm2.text
self.prices.append(data)
return len(self.prices) > 0
def get_data(self, **params):
if self.report is None:
raise Exception("ElexonAPI objects require the report be set before use.")
if self.apikey is None:
raise Exception("An API key is required to use the Elexon API accessor functionality")
url = make_elexon_url(self.report, self.version)
params.update({'APIKey': self.apikey, 'ServiceType': 'xml'})
req = get_or_post_a_url(url, params=params)
xml = parse_response_as_xml(req)
http = xml.xpath('/response/responseMetadata/httpCode')
if int(http[0].text) != 200:
return False
for item in xml.xpath('/response/responseBody/responseList/item'):
item_dict = map_children_to_dict(item, self.XML_MAPPING)
self.post_item_cleanup(item_dict)
self.items.append(item_dict)
return True
def get_data(self):
""" Get the report data and update.
:returns: True or False
:rtype: bool
"""
self.data = []
params = {
'param5': self.date.strftime("%Y-%m-%d"),
'param6': self.period,
'element': self.type,
'param1': self.unitid,
'param2': self.unittype,
'param3': self.leadparty,
'param4': self.ngcunitname,
}
if self.historic:
resp = get_or_post_a_url('http://www.bmreports.com/bsp/additional/soapfunctions.php?',
params=params)
return self._process(resp)
return False
def submit(self):
""" Submit the form data and update based on response.
Given how slow the parsing of a 3M HTML page is, try and use the
X-MicrosoftAjax: Delta=true header to get smaller blocks for processing.
"""
is_set, upd = self.form_data.set_value_by_label('Page Size', '25')
if is_set is False:
return False
response = self._do_post(True)
if response is None:
return False
if self.form_data.update(response.content) is False:
self.logger.warning("Submit failed :-(")
return False
if self.form_data.export_url is None:
self.logger.warning("Unable to find the export url. Cannot continue.")
return False
export_url = _make_url(self.form_data.export_url) + 'XML'
response = get_or_post_a_url(export_url, cookies=self.cookies)
self.raw_data = response.content
return True
def get_data(self):
""" Get the report data and update.
:returns: True or False
:rtype: bool
"""
self.data = []
params = {
'param5': self.date.strftime("%Y-%m-%d"),
'param6': self.period,
'element': self.type,
'param1': self.unitid,
'param2': self.unittype,
'param3': self.leadparty,
'param4': self.ngcunitname,
}
if self.historic:
resp = get_or_post_a_url(
'http://www.bmreports.com/bsp/additional/soapfunctions.php?',
params=params)
return self._process(resp)
return False
def get_list(self):
""" Download and update the unit list.
:rtype: bool
"""
self.units = []
resp = get_or_post_a_url(self.XLS_URL)
self.raw_data = resp.content
tmp_f = NamedTemporaryFile(delete=False)
with open(tmp_f.name, 'wb') as fhh:
fhh.write(resp.content)
wbb = xlrd.open_workbook(tmp_f.name)
sht = wbb.sheet_by_name(self.SHEET_NAME)
for rownum in range(1, sht.nrows):
self._extract_row_data(wbb, sht, rownum)
try:
os.unlink(tmp_f.name)
except Exception:
pass
return True
def submit(self):
""" Submit the form data and update based on response.
Given how slow the parsing of a 3M HTML page is, try and use the
X-MicrosoftAjax: Delta=true header to get smaller blocks for processing.
"""
is_set, upd = self.form_data.set_value_by_label('Page Size', '25')
if is_set is False:
return False
response = self._do_post(True)
if response is None:
return False
if self.form_data.update(response.content) is False:
self.logger.warning("Submit failed :-(")
return False
if self.form_data.export_url is None:
self.logger.warning(
"Unable to find the export url. Cannot continue.")
return False
export_url = _make_url(self.form_data.export_url) + 'XML'
response = get_or_post_a_url(export_url, cookies=self.cookies)
self.raw_data = response.content
return True
def get_list(self):
""" Download and update the unit list.
:rtype: bool
"""
self.units = []
resp = get_or_post_a_url(self.XLS_URL)
self.raw_data = resp.content
tmp_f = NamedTemporaryFile(delete=False)
with open(tmp_f.name, 'wb') as fhh:
fhh.write(resp.content)
wbb = xlrd.open_workbook(tmp_f.name)
sht = wbb.sheet_by_name(self.SHEET_NAME)
for rownum in range(1, sht.nrows):
self._extract_row_data(wbb, sht, rownum)
try:
os.unlink(tmp_f.name)
except Exception:
pass
return True
