def new_request(url):
# Remove username/password from url.
netloc = '%s:%s' % (url.hostname, url.port or 80)
url = urlunsplit((url.scheme, netloc, url.path, url.query,
url.fragment)).rstrip('?&')
log = logging.getLogger('pydap')
log.INFO('Opening %s' % url)
r = urllib2.urlopen(url)
# Detect redirection.
if r.url != url:
data = r.read()
code = BeautifulSoup(data)
# Check if we need to authenticate:
if code.find('form'):
# Ok, we need to authenticate. Let's get the location
# where we need to POST the information.
post_location = code.find('form').get('action', r.url)
# Do a post, passing our credentials.
inputs = code.find('form').findAll('input')
params = dict([(el['name'], el['value']) for el in inputs
if el['type'] == 'hidden'])
params[username_field] = url.username
params[password_field] = url.password
params = urllib.urlencode(params)
req = urllib2.Request(post_location, params)
r = urllib2.urlopen(req)
# Parse the response.
data = r.read()
code = BeautifulSoup(data)
# Get the location from the Javascript code. Depending on the
# CAS this code has to be changed. Ideally, the server would
# redirect with HTTP headers and this wouldn't be necessary.
script = code.find('script').string
redirect = re.search('window.location.href="(.*)"',
script).group(1)
r = urllib2.urlopen(redirect)
resp = r.headers.dict
resp['status'] = str(r.code)
data = r.read()
# When an error is returned, we parse the error message from the
# server and return it in a ``ClientError`` exception.
if resp.get("content-description") == "dods_error":
m = re.search('code = (?P<code>\d+);\s*message = "(?P<msg>.*)"',
data, re.DOTALL | re.MULTILINE)
msg = 'Server error %(code)s: "%(msg)s"' % m.groupdict()
raise ClientError(msg)
return resp, data
def new_request(url):
if url[-1] is '&': url = url[0:-1]
# log.debug('Opening %s (install_basic_client)' % url)
r = urllib.request.urlopen(url)
resp = r.headers.dict
resp['status'] = str(r.code)
data = r.read()
# When an error is returned, we parse the error message from the
# server and return it in a ``ClientError`` exception.
if resp.get("content-description") == "dods_error":
m = re.search('code = (?P<code>\d+);\s*message = "(?P<msg>.*)"', data, re.DOTALL | re.MULTILINE)
msg = 'Server error %(code)s: "%(msg)s"' % m.groupdict()
raise ClientError(msg)
return resp, data
def open_url(url):
"""
Open a given dataset URL, trying different response methods.
The function checks the stub DDX method, and falls back to the
DDS+DAS responses. It can be easily extended for other representations
like JSON.
The URL should point to the dataset, omitting any response extensions
like ``.dds``. Username and password can be passed in the URL like::
http://user:[email protected]:port/path
They will be transmitted as plaintext if the server supports only
Basic authentication, so be careful. For Digest authentication this
is safe.
The URL can point directly to an Opendap dataset, or it can contain
any number of contraint expressions (selection/projections)::
http://example.com/dataset?var1,var2&var3>10
You can also specify a cache directory, a timeout and a proxy using
the global variables from ``pydap.lib``::
>>> import pydap.lib
>>> pydap.lib.TIMEOUT = 60 # seconds
>>> pydap.lib.CACHE = '.cache'
>>> import httplib2
>>> from pydap.util import socks
>>> pydap.lib.PROXY = httplib2.ProxyInfo(socks.PROXY_TYPE_HTTP, 'localhost', 8000)
"""
for response in [_ddx, _ddsdas]:
dataset = response(url)
if dataset: break
else:
raise ClientError("Unable to open dataset.")
# Remove any projections from the url, leaving selections.
scheme, netloc, path, query, fragment = urlsplit(url)
projection, selection = parse_qs(query)
url = urlunsplit(
(scheme, netloc, path, '&'.join(selection), fragment))
# Set data to a Proxy object for BaseType and SequenceType. These
# variables can then be sliced to retrieve the data on-the-fly.
for var in walk(dataset, BaseType):
var.data = ArrayProxy(var.id, url, var.shape)
for var in walk(dataset, SequenceType):
var.data = SequenceProxy(var.id, url)
# Set server-side functions.
dataset.functions = Functions(url)
# Apply the corresponding slices.
projection = fix_shn(projection, dataset)
for var in projection:
target = dataset
while var:
token, slice_ = var.pop(0)
target = target[token]
if slice_ and isinstance(target.data, VariableProxy):
shape = getattr(target, 'shape', (sys.maxint,))
target.data._slice = fix_slice(slice_, shape)
return dataset