def parse(self, stations, years, fields):
''' Pass in some stations and years. For convenience (and to match CRN's data output)
we'll only deal with data in complete years. '''
# First make sure we've got the data locally:
self._download(stations, years)
doc = DataObjectCollection()
for station in stations:
do = DataObject()
for field in self.fields:
# Skip some fields we know we don't care about
useless_fields = ['WBANNO', 'UTC_DATE', 'UTC_TIME', 'LST_DATE', 'LST_TIME', 'CRX_VN', 'SUR_TEMP_TYPE']
if (fields and field not in fields) or (not fields and field in useless_fields):
continue
do[field] = TimeSeries([])
for year in years:
f = open(self.storage_dir + self._filename(station, year))
for line in f:
values = line.split()
do.append(values, self.fields)
for ts in do.values():
ts.replace_data(interpolate_forward_backward(ts, missing_values))
doc.append(do)
return doc
def parse(self, stations, years, fields):
''' Pass in some stations and years. For convenience (and to match CRN's data output)
we'll only deal with data in complete years. '''
# First make sure we've got the data locally:
self._download(stations, years)
doc = DataObjectCollection()
for station in stations:
do = DataObject()
for field in self.fields:
# Skip some fields we know we don't care about
useless_fields = [
'WBANNO', 'UTC_DATE', 'UTC_TIME', 'LST_DATE', 'LST_TIME',
'CRX_VN', 'SUR_TEMP_TYPE'
]
if (fields and field not in fields) or (not fields and field
in useless_fields):
continue
do[field] = TimeSeries([])
for year in years:
f = open(self.storage_dir + self._filename(station, year))
for line in f:
values = line.split()
do.append(values, self.fields)
for ts in do.values():
ts.replace_data(
interpolate_forward_backward(ts, missing_values))
doc.append(do)
return doc
def parse(self, listofdicts):
doc = DataObjectCollection()
for curdict in listofdicts:
do = DataObject()
for key, val in curdict.items():
do[key] = val
doc.append(do)
return doc
def test_datamapper_1():
# create a TimeSeries and stick something in it
ts1 = TimeSeries(['datapoint'], sample_rate=60)
# create a DO and put the TS in it
do1 = DataObject()
do1['somedata'] = ts1
assert do1.keys() == ['somedata']
# create a DOC and put the DO in it
doc = DataObjectCollection()
doc.append(do1)
# dig down through the levels and get the datapoint we originally inserted
timeseries = doc[0]['somedata']
datapoint = timeseries[0]
assert(datapoint == 'datapoint')
def parse(self, sineslist):
doc = DataObjectCollection()
for sines in sineslist:
do = DataObject()
for key, sine in sines.items():
ts = TimeSeries(sine)
ts.sample_rate = 1
do[key] = ts
doc.append(do)
return doc
def parse(self, sines):
doc = DataObjectCollection()
do = DataObject()
for key, sine in sines.items():
ts = TimeSeries(sine)
ts.sample_rate = 1
# ts.rangex = (-1,1)
do[key] = ts
doc.append(do)
return doc
def parse(self):
d = eval(self.file.readline().strip())
doc = DataObjectCollection(sample_rate=1 / 3.0)
for i, octant in enumerate(d):
do = DataObject()
for varname, values in octant.items():
ts = TimeSeries(values)
do[varname] = ts
doc.append(do)
return doc
def test_doc_imposes_sample_rate():
# create a DO
do1 = DataObject()
# create a DOC and put the DO in it
doc = DataObjectCollection(sample_rate=60)
doc.append(do1)
retrieved_do = doc[0]
assert(retrieved_do.sample_rate == 60)
def test_do_imposes_sample_rate():
# create a TimeSeries
ts1 = TimeSeries(['datapoint'])
# create a DO and put the TS in it
do1 = DataObject(sample_rate=60)
do1['somed'] = ts1
# test the sample_rate of the TimeSeries (which it should derive from the DO)
assert(do1['somed'].sample_rate == 60)
# while we're at it, make sure that it percolates up to the DOC
doc = DataObjectCollection([do1])
assert doc.sample_rate == 60, str(doc.sample_rate) + ' is not 60.'
def _getDataObject():
''' Convenience method to return a DataObject initialized to fit the buoy data. '''
do = DataObject(metadata={'buoy_id': id})
for key in ['LAT', 'LON', 'TEMP']:
do[key] = TimeSeries([])
return do
