def __init__(self, url):
# download DDS/DAS
scheme, netloc, path, query, fragment = urlsplit(url)
ddsurl = urlunsplit((scheme, netloc, path + '.dds', query, fragment))
r = requests.get(ddsurl)
r.raise_for_status()
dds = r.text.encode('utf-8')
dasurl = urlunsplit((scheme, netloc, path + '.das', query, fragment))
r = requests.get(dasurl)
r.raise_for_status()
das = r.text.encode('utf-8')
# build the dataset from the DDS and add attributes from the DAS
self.dataset = build_dataset(dds)
add_attributes(self.dataset, parse_das(das))
# remove any projection from the url, leaving selections
projection, selection = parse_ce(query)
url = urlunsplit((scheme, netloc, path, '&'.join(selection), fragment))
# now add data proxies
for var in walk(self.dataset, BaseType):
var.data = BaseProxy(url, var.id, var.descr)
for var in walk(self.dataset, SequenceType):
var.data = SequenceProxy(url, var.id, var.descr)
# apply projections
for var in projection:
target = self.dataset
while var:
token, index = var.pop(0)
target = target[token]
if index and isinstance(target.data, BaseProxy):
target.data.slice = fix_slice(index, target.shape)
def open_file(dods, das=None):
"""Open a file downloaded from a `.dods` response, returning a dataset.
Optionally, read also the `.das` response to assign attributes to the
dataset.
"""
dds = ''
# This file contains both ascii _and_ binary data
# Let's handle them separately in sequence
# Without ignoring errors, the IO library will
# actually read past the ascii part of the
# file (despite our break from iteration) and
# will error out on the binary data
with open(dods, "rt", buffering=1, encoding='ascii',
newline='\n', errors='ignore') as f:
for line in f:
if line.strip() == 'Data:':
break
dds += line
dataset = build_dataset(dds)
pos = len(dds) + len('Data:\n')
with open(dods, "rb") as f:
f.seek(pos)
dataset.data = unpack_data(f, dataset)
if das is not None:
with open(das) as f:
add_attributes(dataset, parse_das(f.read()))
return dataset
def __init__(self, url):
# download DDS/DAS
scheme, netloc, path, query, fragment = urlsplit(url)
ddsurl = urlunsplit((scheme, netloc, path + '.dds', query, fragment))
dds = requests.get(ddsurl).text.encode('utf-8')
dasurl = urlunsplit((scheme, netloc, path + '.das', query, fragment))
das = requests.get(dasurl).text.encode('utf-8')
# build the dataset from the DDS and add attributes from the DAS
self.dataset = build_dataset(dds)
add_attributes(self.dataset, parse_das(das))
# remove any projection from the url, leaving selections
projection, selection = parse_ce(query)
url = urlunsplit((scheme, netloc, path, '&'.join(selection), fragment))
# now add data proxies
for var in walk(self.dataset, BaseType):
var.data = BaseProxy(url, var.id, var.descr)
for var in walk(self.dataset, SequenceType):
var.data = SequenceProxy(url, var.id, var.descr)
# apply projections
for var in projection:
target = self.dataset
while var:
token, index = var.pop(0)
target = target[token]
if index and isinstance(target.data, BaseProxy):
target.data.slice = fix_slice(index, target.shape)
def __init__(self, url, application=None, session=None, output_grid=True):
# download DDS/DAS
scheme, netloc, path, query, fragment = urlsplit(url)
ddsurl = urlunsplit((scheme, netloc, path + '.dds', query, fragment))
r = GET(ddsurl, application, session)
raise_for_status(r)
dds = r.text
dasurl = urlunsplit((scheme, netloc, path + '.das', query, fragment))
r = GET(dasurl, application, session)
raise_for_status(r)
das = r.text
# build the dataset from the DDS and add attributes from the DAS
self.dataset = build_dataset(dds)
add_attributes(self.dataset, parse_das(das))
# remove any projection from the url, leaving selections
projection, selection = parse_ce(query)
url = urlunsplit((scheme, netloc, path, '&'.join(selection), fragment))
# now add data proxies
for var in walk(self.dataset, BaseType):
var.data = BaseProxy(url,
var.id,
var.dtype,
var.shape,
application=application,
session=session)
for var in walk(self.dataset, SequenceType):
template = copy.copy(var)
var.data = SequenceProxy(url,
template,
application=application,
session=session)
# apply projections
for var in projection:
target = self.dataset
while var:
token, index = var.pop(0)
target = target[token]
if isinstance(target, BaseType):
target.data.slice = fix_slice(index, target.shape)
elif isinstance(target, GridType):
index = fix_slice(index, target.array.shape)
target.array.data.slice = index
for s, child in zip(index, target.maps):
target[child].data.slice = (s, )
elif isinstance(target, SequenceType):
target.data.slice = index
# retrieve only main variable for grid types:
for var in walk(self.dataset, GridType):
var.set_output_grid(output_grid)
def open_file(dods, das=None):
with open(dods) as f:
dds, data = f.read().split('\nData:\n', 1)
dataset = build_dataset(dds)
dataset.data = unpack_data(data, dataset)
if das is not None:
with open(das) as f:
add_attributes(dataset, parse_das(f.read()))
return dataset
def open_dods(url, metadata=False):
r = requests.get(url)
dds, data = r.content.split('\nData:\n', 1)
dataset = build_dataset(dds)
dataset.data = unpack_data(data, dataset)
if metadata:
scheme, netloc, path, query, fragment = urlsplit(url)
dasurl = urlunsplit((scheme, netloc, path[:-4] + 'das', query, fragment))
das = requests.get(dasurl).text.encode('utf-8')
add_attributes(dataset, parse_das(das))
return dataset
def __init__(self, url, application=None, session=None, output_grid=True):
# download DDS/DAS
scheme, netloc, path, query, fragment = urlsplit(url)
ddsurl = urlunsplit((scheme, netloc, path + ".dds", query, fragment))
r = GET(ddsurl, application, session)
raise_for_status(r)
dds = r.text
dasurl = urlunsplit((scheme, netloc, path + ".das", query, fragment))
r = GET(dasurl, application, session)
raise_for_status(r)
das = r.text
# build the dataset from the DDS and add attributes from the DAS
self.dataset = build_dataset(dds)
add_attributes(self.dataset, parse_das(das))
# remove any projection from the url, leaving selections
projection, selection = parse_ce(query)
url = urlunsplit((scheme, netloc, path, "&".join(selection), fragment))
# now add data proxies
for var in walk(self.dataset, BaseType):
var.data = BaseProxy(url, var.id, var.dtype, var.shape, application=application, session=session)
for var in walk(self.dataset, SequenceType):
template = copy.copy(var)
var.data = SequenceProxy(url, template, application=application, session=session)
# apply projections
for var in projection:
target = self.dataset
while var:
token, index = var.pop(0)
target = target[token]
if isinstance(target, BaseType):
target.data.slice = fix_slice(index, target.shape)
elif isinstance(target, GridType):
index = fix_slice(index, target.array.shape)
target.array.data.slice = index
for s, child in zip(index, target.maps):
target[child].data.slice = (s,)
elif isinstance(target, SequenceType):
target.data.slice = index
# retrieve only main variable for grid types:
for var in walk(self.dataset, GridType):
var.set_output_grid(output_grid)
def open_dods(url, metadata=False, application=None, session=None):
"""Open a `.dods` response directly, returning a dataset."""
r = GET(url, application, session)
dds, data = r.body.split(b'\nData:\n', 1)
dds = dds.decode(r.content_encoding or 'ascii')
dataset = build_dataset(dds)
stream = StreamReader(BytesIO(data))
dataset.data = unpack_data(stream, dataset)
if metadata:
scheme, netloc, path, query, fragment = urlsplit(url)
dasurl = urlunsplit(
(scheme, netloc, path[:-4] + 'das', query, fragment))
das = GET(dasurl, application, session).text
add_attributes(dataset, parse_das(das))
return dataset
def open_dods(url, metadata=False, application=None, session=None):
"""Open a `.dods` response directly, returning a dataset."""
r = GET(url, application, session)
dds, data = r.body.split(b'\nData:\n', 1)
dds = dds.decode(r.content_encoding or 'ascii')
dataset = build_dataset(dds)
stream = StreamReader(BytesIO(data))
dataset.data = unpack_data(stream, dataset)
if metadata:
scheme, netloc, path, query, fragment = urlsplit(url)
dasurl = urlunsplit(
(scheme, netloc, path[:-4] + 'das', query, fragment))
das = GET(dasurl, application, session).text
add_attributes(dataset, parse_das(das))
return dataset
def open_file(dods, das=None):
"""Open a file downloaded from a `.dods` response, returning a dataset.
Optionally, read also the `.das` response to assign attributes to the
dataset.
"""
dds = ''
# This file contains both ascii _and_ binary data
# Let's handle them separately in sequence
# Without ignoring errors, the IO library will actually
# read past the ascii part of the
# file (despite our break from iteration) and will error
# out on the binary data
with open(dods,
"rt",
buffering=1,
encoding='ascii',
newline='\n',
errors='ignore') as f:
for line in f:
if line.strip() == 'Data:':
break
dds += line
dataset = build_dataset(dds)
pos = len(dds) + len('Data:\n')
with open(dods, "rb") as f:
f.seek(pos)
dataset.data = unpack_data(f, dataset)
if das is not None:
with open(das) as f:
add_attributes(dataset, parse_das(f.read()))
return dataset
class CSVHandler(BaseHandler):
"""This is a simple handler for CSV files."""
__version__ = get_distribution("pydap.handlers.csv").version
extensions = re.compile(r"^.*\.csv$", re.IGNORECASE)
def __init__(self, filepath):
BaseHandler.__init__(self)
try:
with open(filepath, 'Ur') as fp:
reader = csv.reader(fp, quoting=csv.QUOTE_NONNUMERIC)
vars = reader.next()
except Exception, exc:
message = 'Unable to open file {filepath}: {exc}'.format(
filepath=filepath, exc=exc)
raise OpenFileError(message)
self.additional_headers.append(
('Last-modified',
(formatdate(
time.mktime(
time.localtime(os.stat(filepath)[ST_MTIME]))))))
# build dataset
name = os.path.split(filepath)[1]
self.dataset = DatasetType(name)
# add sequence and children for each column
seq = self.dataset['sequence'] = SequenceType('sequence')
for var in vars:
seq[var] = BaseType(var)
# set the data
seq.data = CSVData(filepath, copy.copy(seq))
# add extra attributes
metadata = "{0}.json".format(filepath)
if os.path.exists(metadata):
with open(metadata) as fp:
attributes = json.load(fp)
add_attributes(self.dataset, attributes)
def setUp(self):
"""Load a dataset and apply DAS to it."""
self.dataset = build_dataset(DDS)
attributes = parse_das(DAS)
add_attributes(self.dataset, attributes)
def setUp(self):
dataset = build_dataset(DDS)
attributes = parse_das(DAS)
self.dataset = add_attributes(dataset, attributes)
def setUp(self):
"""Load a dataset and apply DAS to it."""
self.dataset = build_dataset(DDS)
attributes = parse_das(DAS)
add_attributes(self.dataset, attributes)
def setUp(self):
dataset = build_dataset(DDS)
attributes = parse_das(DAS)
self.dataset = add_attributes(dataset, attributes)
