def set_datasets(self, start_date, end_date, station, cadence, service):
"""
List of datasets for the POST request form data.
The total span of data will be adjusted based on `cadence`
so that both `start_date` and `end_date` are included.
Total data span downloaded is likely to be
longer than `end_date - start_date`
Parameters
----------
start_date: datetime.date
earliest date at which data wanted.
end_date: datetime.datetime
latest date at which data wanted.
station: string
IAGA-style station code e.g. 'ESK', 'NGK'
cadence: string
frequency of the data. 'minute' or 'hour',
changes the total data span
service: string
webservice to target, only 'WDC' for now
(+ 'INTERMAGNET' in future)
Returns
-------
datasets for the POST request form in a a comma-seperated
string.
Raises
------
ValueError if `cadence` is not either 'minute' or 'hour'
"""
root = '/'.join(['', service.lower(), 'datasets', cadence, ''])
cadences_supported = ['minute', 'hour']
if cadence == 'hour':
base = root + station.lower() + '{:d}'
years = range(start_date.year, end_date.year + 1)
dsets = (safe_format(base, year) for year in years)
elif cadence == 'minute':
base = root + station.lower() + '{:d}{:02d}'
# note the creation of per-diem date stamps followed by a
# set comprehension over their strings appears wasteful because
# it creates ~30X more date objects than we strictly need.
# but doing the maths correctly ourselves
# including edge and corner cases is
# messy, complex, and easy to screw up
# +1 so we _include_ the end date in range
num_days = (end_date - start_date).days + 1
all_days = (start_date + timedelta(day) for day in range(num_days))
dsets = {safe_format(base, dt.year, dt.month) for dt in all_days}
else:
mess = 'cadence {} cannot be handled.\nShould be one of: {}'
raise ValueError(safe_format(mess, cadence, cadences_supported))
self.datasets = ','.join(dset for dset in dsets)
def extract_url(self):
"""
Form the URL to send the request to
by reading the ConfigurationParser `config`
Returns
-------
url for request as a `str`
Raises
------
ConfigError if any of the required parts of the url are not options
within the config file
"""
try:
url = '/'.join(
self._config.get(self.service, k) for k in self.urlbits_need)
except NoOptionError as err:
mess = ('cannot load request url from config\n' +
'require values for {0}\n' +
'under section for service `[{1}]`\n' +
'found only {2}\n' + str(err))
formatted = safe_format(mess, self.urlbits_need, self.service,
list(self._config[self.service].keys()))
raise ConfigError(formatted)
return url
def extract_headers(self):
"""
Get the request headers from the ConfigurationParser `config`.
`service` is assumed to be a section in the `config`
Returns
-------
`dict` of headers for the request
Raises
------
ConfigError if any of the required header values
are not options within the config file
"""
try:
heads = {
k: self._config.get(self.service, k)
for k in self.headers_need
}
except NoOptionError as err:
mess = ('cannot load request headers from config\n' +
'require values for {0}\n' +
'under section for service `[{1}]`\n' +
'found only {2}\n' + str(err))
formatted_mess = safe_format(
mess, self.headers_need, self.service,
list(self._config[self.service].keys()))
raise ConfigError(formatted_mess)
return heads
def form_data__format(self):
"""
The format for the output files as read from the config.
N.B.
The double_underscore (`__`) in the name is trying to indicate
that we are reading the 'format' part of the 'form_data'
from the config file.... inspired by
[django's usage](http://stackoverflow.com/questions/5481682)
Returns
-------
dataformat: string:
What format do we want the response data to be in
when we make a request?
Raises
------
ConfigError if cannot make valid data format selection
from options within the config file
"""
template_option = '_format_template'
outfile_option = 'FileFormat'
try:
outfmt_template = self._config.get(self.service, template_option)
except NoOptionError:
mess = ('cannot find required value {}\n' +
'in config for service:{}')
formatted_mess = safe_format(mess, template_option, self.service)
raise ConfigError(formatted_mess)
try:
outfiletype = self._config.get(self.service, outfile_option)
except NoOptionError:
mess = ('cannot find "FileType" option value {}\n' +
'in config for service:{}')
formatted_mess = safe_format(mess, outfile_option, self.service)
raise ConfigError(formatted_mess)
final_format = safe_format(outfmt_template, outfiletype)
return final_format
def _check_service(self, service):
"""
Validate the `service` by ensuring the config file has a section
of options for the `service` we are interested in.
Parameters
----------
service: string
webservice to target, either 'WDC' (or,
in future 'INTERMAGNET')
"""
if service not in self._config.sections():
mess = ('cannot find service:{0} in configuration\n' +
'should look like (like `[{0}]`)\n' +
'found sections for {1}\n')
formatted_mess = safe_format(mess, service,
self._config.sections())
raise ConfigError(formatted_mess)
def __repr__(self):
mess = safe_format('{}({}, {})', self.__class__.__name__,
repr(self._filename), repr(self.service))
return mess
def __repr__(self):
args_part = safe_format('({})', self._from_req_parser)
return self.__class__.__name__ + args_part
def __str__(self):
"""pretty (ish) printing)"""
strout = safe_format('{}:\n {}', self.__class__.__name__,
self._dict)
return strout