def download_url(server, dataset_id, protocol, variables, response='html', constraints=None): """The download URL for the `server` endpoint. Args: dataset_id (str): a dataset unique id. protocol (str): tabledap or griddap. variables (list/tuple): a list of the variables to download. response (str): default is HTML. constraints (dict): download constraints, default None (opendap-like url) example: constraints = {'latitude<=': 41.0, 'latitude>=': 38.0, 'longitude<=': -69.0, 'longitude>=': -72.0, 'time<=': '2017-02-10T00:00:00+00:00', 'time>=': '2016-07-10T00:00:00+00:00', } Returns: url (str): the download URL for the `response` chosen. """ base = '{server_url}/{protocol}/{dataset_id}' if not constraints: url = base.format( server_url=server, protocol=protocol, dataset_id=dataset_id ) else: base += '.{response}?{variables}{constraints}' _constraints = copy.copy(constraints) for k, v in _constraints.items(): if k.startswith('time'): _constraints.update({k: parse_dates(v)}) _constraints = quote_string_constraints(_constraints) _constraints = ''.join(['&{}{}'.format(k, v) for k, v in _constraints.items()]) variables = ','.join(variables) url = base.format( server_url=server, protocol=protocol, dataset_id=dataset_id, response=response, variables=variables, constraints=_constraints ) return _check_url_response(url)
def url(self, response=None): """ Return the URL used to download data """ # Replace erddapy get_download_url # We need to replace it to better handle http responses with by-passing the _check_url_response # https://github.com/ioos/erddapy/blob/fa1f2c15304938cd0aa132946c22b0427fd61c81/erddapy/erddapy.py#L247 # Define constraint to select this box of data: self.define_constraints() # This will affect self.erddap.constraints # Define the list of variables to retrieve - all for the index self.erddap.variables = [ 'file', 'date', 'longitude', 'latitude', 'ocean', 'profiler_type', 'institution', 'date_update' ] # dataset_id = self.erddap.dataset_id protocol = self.erddap.protocol variables = self.erddap.variables if not response: response = self.erddap.response constraints = self.erddap.constraints url = f"{self.erddap.server}/{protocol}/{dataset_id}.{response}?" if variables: variables = ",".join(variables) url += f"{variables}" if constraints: _constraints = copy.copy(constraints) for k, v in _constraints.items(): if k.startswith("time"): _constraints.update({k: parse_dates(v)}) _constraints = quote_string_constraints(_constraints) _constraints = "".join( [f"&{k}{v}" for k, v in _constraints.items()]) url += f"{_constraints}" url += '&distinct()&orderBy("date")' # In erddapy: # url = _distinct(url, **kwargs) # return _check_url_response(url, **self.requests_kwargs) return url
def get_url(self): """ Return the URL to download data requested Returns ------- str """ # Replace erddapy get_download_url() # We need to replace it to better handle http responses with by-passing the _check_url_response # https://github.com/ioos/erddapy/blob/fa1f2c15304938cd0aa132946c22b0427fd61c81/erddapy/erddapy.py#L247 # First part of the URL: protocol = self.erddap.protocol dataset_id = self.erddap.dataset_id response = self.erddap.response url = f"{self.erddap.server}/{protocol}/{dataset_id}.{response}?" # Add variables to retrieve: self.erddap.variables = (self._minimal_vlist ) # Define the list of variables to retrieve variables = self.erddap.variables variables = ",".join(variables) url += f"{variables}" # Add constraints: self.define_constraints( ) # Define constraint to select this box of data (affect self.erddap.constraints) constraints = self.erddap.constraints _constraints = copy.copy(constraints) for k, v in _constraints.items(): if k.startswith("time"): _constraints.update({k: parse_dates(v)}) _constraints = quote_string_constraints(_constraints) _constraints = "".join([f"&{k}{v}" for k, v in _constraints.items()]) url += f"{_constraints}" # Last part: url += '&distinct()&orderBy("time,pres")' return url
def test_quote_string_constraints(): """Ensure that only string are quoted.""" kw = quote_string_constraints({ 'latitude': 42, 'longitude': 42., 'max_time': datetime.utcnow(), 'min_time': '1970-01-01T00:00:00Z', 'cdm_data_type': 'trajectoryprofile', }) assert isinstance(kw['latitude'], int) assert isinstance(kw['longitude'], float) assert isinstance(kw['max_time'], datetime) assert isinstance(kw['min_time'], str) assert isinstance(kw['cdm_data_type'], str) assert kw['min_time'].startswith('"') and kw['min_time'].endswith('"') assert kw['cdm_data_type'].startswith( '"') and kw['cdm_data_type'].endswith('"') for k, v in kw.items(): if isinstance(v, str): assert v.startswith('"') and v.endswith('"')
def test_quote_string_constraints(): """Ensure that only string are quoted.""" kw = quote_string_constraints({ "latitude": 42, "longitude": 42.0, "max_time": datetime.utcnow(), "min_time": "1970-01-01T00:00:00Z", "cdm_data_type": "trajectoryprofile", }) assert isinstance(kw["latitude"], int) assert isinstance(kw["longitude"], float) assert isinstance(kw["max_time"], datetime) assert isinstance(kw["min_time"], str) assert isinstance(kw["cdm_data_type"], str) assert kw["min_time"].startswith('"') and kw["min_time"].endswith('"') assert kw["cdm_data_type"].startswith( '"') and kw["cdm_data_type"].endswith('"') for k, v in kw.items(): if isinstance(v, str): assert v.startswith('"') and v.endswith('"')
def get_download_url( self, dataset_id=None, protocol=None, variables=None, response=None, constraints=None, ): """The download URL for the `server` endpoint. Args: dataset_id (str): a dataset unique id. protocol (str): tabledap or griddap. variables (list/tuple): a list of the variables to download. response (str): default is HTML. constraints (dict): download constraints, default None (opendap-like url) example: constraints = {'latitude<=': 41.0, 'latitude>=': 38.0, 'longitude<=': -69.0, 'longitude>=': -72.0, 'time<=': '2017-02-10T00:00:00+00:00', 'time>=': '2016-07-10T00:00:00+00:00',} Returns: url (str): the download URL for the `response` chosen. """ dataset_id = dataset_id if dataset_id else self.dataset_id protocol = protocol if protocol else self.protocol variables = variables if variables else self.variables response = response if response else self.response constraints = constraints if constraints else self.constraints if not dataset_id: raise ValueError( f"Please specify a valid `dataset_id`, got {dataset_id}") if not protocol: raise ValueError( f"Please specify a valid `protocol`, got {protocol}") # This is an unconstrained OPeNDAP response b/c # the integer based constrained version is just not worth supporting ;-p if response == "opendap": return f"{self.server}/{protocol}/{dataset_id}" else: url = f"{self.server}/{protocol}/{dataset_id}.{response}?" if variables: variables = ",".join(variables) url += f"{variables}" if constraints: _constraints = copy.copy(constraints) for k, v in _constraints.items(): if k.startswith("time"): _constraints.update({k: parse_dates(v)}) _constraints = quote_string_constraints(_constraints) _constraints = "".join( [f"&{k}{v}" for k, v in _constraints.items()]) url += f"{_constraints}" return _check_url_response(url)