def download_url(server, dataset_id, protocol, variables, response='html', constraints=None):
"""The download URL for the `server` endpoint.
Args:
dataset_id (str): a dataset unique id.
protocol (str): tabledap or griddap.
variables (list/tuple): a list of the variables to download.
response (str): default is HTML.
constraints (dict): download constraints, default None (opendap-like url)
example: constraints = {'latitude<=': 41.0,
'latitude>=': 38.0,
'longitude<=': -69.0,
'longitude>=': -72.0,
'time<=': '2017-02-10T00:00:00+00:00',
'time>=': '2016-07-10T00:00:00+00:00',
}
Returns:
url (str): the download URL for the `response` chosen.
"""
base = '{server_url}/{protocol}/{dataset_id}'
if not constraints:
url = base.format(
server_url=server,
protocol=protocol,
dataset_id=dataset_id
)
else:
base += '.{response}?{variables}{constraints}'
_constraints = copy.copy(constraints)
for k, v in _constraints.items():
if k.startswith('time'):
_constraints.update({k: parse_dates(v)})
_constraints = quote_string_constraints(_constraints)
_constraints = ''.join(['&{}{}'.format(k, v) for k, v in _constraints.items()])
variables = ','.join(variables)
url = base.format(
server_url=server,
protocol=protocol,
dataset_id=dataset_id,
response=response,
variables=variables,
constraints=_constraints
)
return _check_url_response(url)
def url(self, response=None):
""" Return the URL used to download data
"""
# Replace erddapy get_download_url
# We need to replace it to better handle http responses with by-passing the _check_url_response
# https://github.com/ioos/erddapy/blob/fa1f2c15304938cd0aa132946c22b0427fd61c81/erddapy/erddapy.py#L247
# Define constraint to select this box of data:
self.define_constraints() # This will affect self.erddap.constraints
# Define the list of variables to retrieve - all for the index
self.erddap.variables = [
'file', 'date', 'longitude', 'latitude', 'ocean', 'profiler_type',
'institution', 'date_update'
]
#
dataset_id = self.erddap.dataset_id
protocol = self.erddap.protocol
variables = self.erddap.variables
if not response:
response = self.erddap.response
constraints = self.erddap.constraints
url = f"{self.erddap.server}/{protocol}/{dataset_id}.{response}?"
if variables:
variables = ",".join(variables)
url += f"{variables}"
if constraints:
_constraints = copy.copy(constraints)
for k, v in _constraints.items():
if k.startswith("time"):
_constraints.update({k: parse_dates(v)})
_constraints = quote_string_constraints(_constraints)
_constraints = "".join(
[f"&{k}{v}" for k, v in _constraints.items()])
url += f"{_constraints}"
url += '&distinct()&orderBy("date")'
# In erddapy:
# url = _distinct(url, **kwargs)
# return _check_url_response(url, **self.requests_kwargs)
return url
def get_url(self):
""" Return the URL to download data requested
Returns
-------
str
"""
# Replace erddapy get_download_url()
# We need to replace it to better handle http responses with by-passing the _check_url_response
# https://github.com/ioos/erddapy/blob/fa1f2c15304938cd0aa132946c22b0427fd61c81/erddapy/erddapy.py#L247
# First part of the URL:
protocol = self.erddap.protocol
dataset_id = self.erddap.dataset_id
response = self.erddap.response
url = f"{self.erddap.server}/{protocol}/{dataset_id}.{response}?"
# Add variables to retrieve:
self.erddap.variables = (self._minimal_vlist
) # Define the list of variables to retrieve
variables = self.erddap.variables
variables = ",".join(variables)
url += f"{variables}"
# Add constraints:
self.define_constraints(
) # Define constraint to select this box of data (affect self.erddap.constraints)
constraints = self.erddap.constraints
_constraints = copy.copy(constraints)
for k, v in _constraints.items():
if k.startswith("time"):
_constraints.update({k: parse_dates(v)})
_constraints = quote_string_constraints(_constraints)
_constraints = "".join([f"&{k}{v}" for k, v in _constraints.items()])
url += f"{_constraints}"
# Last part:
url += '&distinct()&orderBy("time,pres")'
return url
def test_quote_string_constraints():
"""Ensure that only string are quoted."""
kw = quote_string_constraints({
'latitude': 42,
'longitude': 42.,
'max_time': datetime.utcnow(),
'min_time': '1970-01-01T00:00:00Z',
'cdm_data_type': 'trajectoryprofile',
})
assert isinstance(kw['latitude'], int)
assert isinstance(kw['longitude'], float)
assert isinstance(kw['max_time'], datetime)
assert isinstance(kw['min_time'], str)
assert isinstance(kw['cdm_data_type'], str)
assert kw['min_time'].startswith('"') and kw['min_time'].endswith('"')
assert kw['cdm_data_type'].startswith(
'"') and kw['cdm_data_type'].endswith('"')
for k, v in kw.items():
if isinstance(v, str):
assert v.startswith('"') and v.endswith('"')
def test_quote_string_constraints():
"""Ensure that only string are quoted."""
kw = quote_string_constraints({
"latitude": 42,
"longitude": 42.0,
"max_time": datetime.utcnow(),
"min_time": "1970-01-01T00:00:00Z",
"cdm_data_type": "trajectoryprofile",
})
assert isinstance(kw["latitude"], int)
assert isinstance(kw["longitude"], float)
assert isinstance(kw["max_time"], datetime)
assert isinstance(kw["min_time"], str)
assert isinstance(kw["cdm_data_type"], str)
assert kw["min_time"].startswith('"') and kw["min_time"].endswith('"')
assert kw["cdm_data_type"].startswith(
'"') and kw["cdm_data_type"].endswith('"')
for k, v in kw.items():
if isinstance(v, str):
assert v.startswith('"') and v.endswith('"')
def get_download_url(
self,
dataset_id=None,
protocol=None,
variables=None,
response=None,
constraints=None,
):
"""The download URL for the `server` endpoint.
Args:
dataset_id (str): a dataset unique id.
protocol (str): tabledap or griddap.
variables (list/tuple): a list of the variables to download.
response (str): default is HTML.
constraints (dict): download constraints, default None (opendap-like url)
example: constraints = {'latitude<=': 41.0,
'latitude>=': 38.0,
'longitude<=': -69.0,
'longitude>=': -72.0,
'time<=': '2017-02-10T00:00:00+00:00',
'time>=': '2016-07-10T00:00:00+00:00',}
Returns:
url (str): the download URL for the `response` chosen.
"""
dataset_id = dataset_id if dataset_id else self.dataset_id
protocol = protocol if protocol else self.protocol
variables = variables if variables else self.variables
response = response if response else self.response
constraints = constraints if constraints else self.constraints
if not dataset_id:
raise ValueError(
f"Please specify a valid `dataset_id`, got {dataset_id}")
if not protocol:
raise ValueError(
f"Please specify a valid `protocol`, got {protocol}")
# This is an unconstrained OPeNDAP response b/c
# the integer based constrained version is just not worth supporting ;-p
if response == "opendap":
return f"{self.server}/{protocol}/{dataset_id}"
else:
url = f"{self.server}/{protocol}/{dataset_id}.{response}?"
if variables:
variables = ",".join(variables)
url += f"{variables}"
if constraints:
_constraints = copy.copy(constraints)
for k, v in _constraints.items():
if k.startswith("time"):
_constraints.update({k: parse_dates(v)})
_constraints = quote_string_constraints(_constraints)
_constraints = "".join(
[f"&{k}{v}" for k, v in _constraints.items()])
url += f"{_constraints}"
return _check_url_response(url)
