def __iter__(self):
# download and unpack data
r = GET(self.url, self.application, self.session)
raise_for_status(r)
i = r.app_iter
if not hasattr(i, "__next__"):
i = iter(i)
# Fast forward past the DDS header
# the pattern could span chunk boundaries though so make sure to check
previous_chunk = b""
this_chunk = b""
pattern = b"Data:\n"
for this_chunk in i:
m = re.search(pattern, previous_chunk + this_chunk)
if m:
break
if not m:
raise ValueError("Could not find data segment in response from {}".format(self.url))
last_chunk = (previous_chunk + this_chunk)[m.end() :]
# Then construct a stream consisting of everything from
# 'Data:\n' to the end of the chunk + the rest of the stream
def stream_start():
yield last_chunk
stream = StreamReader(chain(stream_start(), i))
return unpack_sequence(stream, self.template)
def __iter__(self):
# download and unpack data
r = GET(self.url, self.application, self.session)
raise_for_status(r)
i = r.app_iter
if not hasattr(i, '__next__'):
i = iter(i)
# Fast forward past the DDS header
# the pattern could span chunk boundaries though so make sure to check
previous_chunk = b''
this_chunk = b''
pattern = b'Data:\n'
for this_chunk in i:
m = re.search(pattern, previous_chunk + this_chunk)
if m:
break
if not m:
raise ValueError(
"Could not find data segment in response from {}".format(
self.url))
last_chunk = (previous_chunk + this_chunk)[m.end():]
# Then construct a stream consisting of everything from
# 'Data:\n' to the end of the chunk + the rest of the stream
def stream_start():
yield last_chunk
stream = StreamReader(chain(stream_start(), i))
return unpack_sequence(stream, self.template)
def __init__(self, url, application=None, session=None, output_grid=True):
# download DDS/DAS
scheme, netloc, path, query, fragment = urlsplit(url)
ddsurl = urlunsplit((scheme, netloc, path + '.dds', query, fragment))
r = GET(ddsurl, application, session)
raise_for_status(r)
dds = r.text
dasurl = urlunsplit((scheme, netloc, path + '.das', query, fragment))
r = GET(dasurl, application, session)
raise_for_status(r)
das = r.text
# build the dataset from the DDS and add attributes from the DAS
self.dataset = build_dataset(dds)
add_attributes(self.dataset, parse_das(das))
# remove any projection from the url, leaving selections
projection, selection = parse_ce(query)
url = urlunsplit((scheme, netloc, path, '&'.join(selection), fragment))
# now add data proxies
for var in walk(self.dataset, BaseType):
var.data = BaseProxy(url,
var.id,
var.dtype,
var.shape,
application=application,
session=session)
for var in walk(self.dataset, SequenceType):
template = copy.copy(var)
var.data = SequenceProxy(url,
template,
application=application,
session=session)
# apply projections
for var in projection:
target = self.dataset
while var:
token, index = var.pop(0)
target = target[token]
if isinstance(target, BaseType):
target.data.slice = fix_slice(index, target.shape)
elif isinstance(target, GridType):
index = fix_slice(index, target.array.shape)
target.array.data.slice = index
for s, child in zip(index, target.maps):
target[child].data.slice = (s, )
elif isinstance(target, SequenceType):
target.data.slice = index
# retrieve only main variable for grid types:
for var in walk(self.dataset, GridType):
var.set_output_grid(output_grid)
def __getitem__(self, index):
# build download url
index = combine_slices(self.slice, fix_slice(index, self.shape))
scheme, netloc, path, query, fragment = urlsplit(self.baseurl)
url = urlunsplit(
(scheme, netloc, path + ".dods", quote(self.id) + hyperslab(index) + "&" + query, fragment)
).rstrip("&")
# download and unpack data
logger.info("Fetching URL: %s" % url)
r = GET(url, self.application, self.session)
raise_for_status(r)
dds, data = r.body.split(b"\nData:\n", 1)
dds = dds.decode(r.content_encoding or "ascii")
if self.shape:
# skip size packing
if self.dtype.char in "SU":
data = data[4:]
else:
data = data[8:]
# calculate array size
shape = tuple(int(np.ceil((s.stop - s.start) / float(s.step))) for s in index)
size = int(np.prod(shape))
if self.dtype == np.byte:
return np.fromstring(data[:size], "B").reshape(shape)
elif self.dtype.char in "SU":
out = []
for word in range(size):
n = np.asscalar(np.fromstring(data[:4], ">I")) # read length
data = data[4:]
out.append(data[:n])
data = data[n + (-n % 4) :]
return np.array([text_type(x.decode("ascii")) for x in out], "S").reshape(shape)
else:
try:
return np.fromstring(data, self.dtype).reshape(shape)
except ValueError as e:
if str(e) == "total size of new array must be unchanged":
# server-side failure.
# it is expected that the user should be mindful of this:
raise RuntimeError(
(
"variable {0} could not be properly "
"retrieved. To avoid this "
"error consider using open_url(..., "
"output_grid=False)."
).format(quote(self.id))
)
else:
raise
def __getitem__(self, index):
# build download url
index = combine_slices(self.slice, fix_slice(index, self.shape))
scheme, netloc, path, query, fragment = urlsplit(self.baseurl)
url = urlunsplit((
scheme, netloc, path + '.dods',
quote(self.id) + hyperslab(index) + '&' + query,
fragment)).rstrip('&')
# download and unpack data
logger.info("Fetching URL: %s" % url)
r = GET(url, self.application, self.session)
raise_for_status(r)
dds, data = r.body.split(b'\nData:\n', 1)
dds = dds.decode(r.content_encoding or 'ascii')
if self.shape:
# skip size packing
if self.dtype.char in 'SU':
data = data[4:]
else:
data = data[8:]
# calculate array size
shape = tuple(
int(np.ceil((s.stop-s.start)/float(s.step))) for s in index)
size = int(np.prod(shape))
if self.dtype == np.byte:
return np.fromstring(data[:size], 'B').reshape(shape)
elif self.dtype.char in 'SU':
out = []
for word in range(size):
n = np.asscalar(np.fromstring(data[:4], '>I')) # read length
data = data[4:]
out.append(data[:n])
data = data[n + (-n % 4):]
return np.array([text_type(x.decode('ascii'))
for x in out], 'S').reshape(shape)
else:
try:
return np.fromstring(data, self.dtype).reshape(shape)
except ValueError as e:
if str(e) == 'total size of new array must be unchanged':
# server-side failure.
# it is expected that the user should be mindful of this:
raise RuntimeError(
('variable {0} could not be properly '
'retrieved. To avoid this '
'error consider using open_url(..., '
'output_grid=False).').format(quote(self.id)))
else:
raise
def __init__(self, url, application=None, session=None, output_grid=True):
# download DDS/DAS
scheme, netloc, path, query, fragment = urlsplit(url)
ddsurl = urlunsplit((scheme, netloc, path + ".dds", query, fragment))
r = GET(ddsurl, application, session)
raise_for_status(r)
dds = r.text
dasurl = urlunsplit((scheme, netloc, path + ".das", query, fragment))
r = GET(dasurl, application, session)
raise_for_status(r)
das = r.text
# build the dataset from the DDS and add attributes from the DAS
self.dataset = build_dataset(dds)
add_attributes(self.dataset, parse_das(das))
# remove any projection from the url, leaving selections
projection, selection = parse_ce(query)
url = urlunsplit((scheme, netloc, path, "&".join(selection), fragment))
# now add data proxies
for var in walk(self.dataset, BaseType):
var.data = BaseProxy(url, var.id, var.dtype, var.shape, application=application, session=session)
for var in walk(self.dataset, SequenceType):
template = copy.copy(var)
var.data = SequenceProxy(url, template, application=application, session=session)
# apply projections
for var in projection:
target = self.dataset
while var:
token, index = var.pop(0)
target = target[token]
if isinstance(target, BaseType):
target.data.slice = fix_slice(index, target.shape)
elif isinstance(target, GridType):
index = fix_slice(index, target.array.shape)
target.array.data.slice = index
for s, child in zip(index, target.maps):
target[child].data.slice = (s,)
elif isinstance(target, SequenceType):
target.data.slice = index
# retrieve only main variable for grid types:
for var in walk(self.dataset, GridType):
var.set_output_grid(output_grid)
def __getitem__(self, index):
# build download url
index = combine_slices(self.slice, fix_slice(index, self.shape))
scheme, netloc, path, query, fragment = urlsplit(self.baseurl)
url = urlunsplit((
scheme, netloc, path + '.dods',
quote(self.id) + hyperslab(index) + '&' + query,
fragment)).rstrip('&')
# download and unpack data
logger.info("Fetching URL: %s" % url)
r = GET(url, self.application, self.session)
raise_for_status(r)
dds, data = r.body.split(b'\nData:\n', 1)
dds = dds.decode(r.content_encoding or 'ascii')
if self.shape:
# skip size packing
if self.dtype.char in 'SU':
data = data[4:]
else:
data = data[8:]
# calculate array size
shape = tuple(
int(np.ceil((s.stop-s.start)/float(s.step))) for s in index)
size = int(np.prod(shape))
if self.dtype == np.byte:
return np.fromstring(data[:size], 'B')
elif self.dtype.char in 'SU':
out = []
for word in range(size):
n = np.fromstring(data[:4], '>I') # read length
data = data[4:]
out.append(data[:n])
data = data[n + (-n % 4):]
return np.array([ text_type(x.decode('ascii')) for x in out ], 'S')
else:
return np.fromstring(data, self.dtype).reshape(shape)
