def test_simple_moving_average(self): points = [1, 2, 3, 4, 5, 6] series = TimeSeries(zip(points, points)) ma = series.moving_average(3).round() self.assertListEqual(ma.points, [ (3, 2), (4, 3), (5, 4), (6, 5) ]) ma = series.moving_average(5).round() self.assertListEqual(ma.points, [ (5, 3), (6, 4) ])
def test_itertimes(self): x = TimeSeries([1, 2, 3, 4],[1, 4, 9, 16]) i = x.itertimes() nextt = next(i) self.assertTrue ( nextt == 1 ) nextt = next(i) self.assertTrue ( nextt == 2 ) self.assertTrue (nextt.dtype == np.int64)
def test024(self): 'TimeSeries.write_to_pi_file writes dict to stream with 12 offset' stream = mock.Stream() obj = TimeSeries.as_dict(self.testdata + "read.PI.timezone.2.xml") TimeSeries.write_to_pi_file(stream, obj, offset=12) target = file(self.testdata + "targetOutput12.xml").read() current = ''.join(stream.content) self.assertEquals(target.strip(), current.strip())
def test_iteritems(self): x = TimeSeries([1, 2, 3, 4],[1, 4, 9, 16]) i = x.iteritems() nextt = next(i) self.assertTrue ( nextt == (1, 1) ) nextt = next(i) self.assertTrue ( nextt == (2, 4) ) self.assertTrue (len(nextt) ==2 )
def test_111(self): 'get_event is defined and equal to __getitem__' obj = TimeSeries(location_id='loc', parameter_id='par') d1 = datetime(1979, 3, 15, 9, 35) ## setting obj[d1] = 1.23 ## checking self.assertEquals(obj._events[d1], obj.get_event(d1))
def test_112(self): 'get_value is defined returns only value, no flags' obj = TimeSeries(location_id='loc', parameter_id='par') d1 = datetime(1979, 3, 15, 9, 35) ## setting obj[d1] = 1.23 ## checking self.assertEquals(obj._events[d1][0], obj.get_value(d1))
def test_024(self): 'getting events of a eventless time series' obj = TimeSeries(location_id='loc', parameter_id='par') d1 = datetime(1979, 3, 15, 9, 35) d3 = datetime(1979, 4, 12, 9, 35) self.assertEqual(0, len(obj.get_events())) self.assertEqual(0, len(obj.get_events(d3))) self.assertEqual(0, len(obj.get_events(d1, d3)))
def test000(self): 'TimeSeries.write_to_pi_file writes list to new file' obj = TimeSeries.as_list(self.testdata + "read.PI.timezone.2.xml") TimeSeries.write_to_pi_file(self.testdata + "current.xml", obj, offset=2) target = file(self.testdata + "targetOutput.xml").read() current = file(self.testdata + "current.xml").read() self.assertEquals(target.strip(), current.strip())
def test030(self): 'TimeSeries.write_to_pi_file appends children to stream' stream = mock.Stream() obj = TimeSeries.as_dict(self.testdata + "read.PI.timezone.2.xml") TimeSeries.write_to_pi_file(stream, obj, offset=0, append=True) target_lines = file(self.testdata + "targetOutput00.xml").readlines()[3:-1] target = ''.join(i.strip() for i in target_lines) current = ''.join(i.strip() for i in ''.join(stream.content).split('\n')) self.assertEquals(target, current)
def test_110(self): 'add_value is defined and equal to __setitem__' obj = TimeSeries(location_id='loc', parameter_id='par') d1 = datetime(1979, 3, 15, 9, 35) ## setting obj.add_value(d1, 1.23) ## checking self.assertEquals(obj._events[d1], obj[d1])
def test_periodic_decomposition(self): series = TimeSeries([ (1, 100), (2, 200), (3, 100), (4, 200), (5, 100) ]) decomposed = series.decompose(2, periodic=True).round() self.assertTrue(isinstance(decomposed, DataFrame)) self.assertEquals(len(decomposed), 3) for series in decomposed.itervalues(): self.assertListEqual(series.timestamps, [1, 2, 3, 4, 5]) self.assertListEqual(decomposed['trend'].values, [150] * 5) self.assertListEqual(decomposed['seasonal'].values, [-50, 50, -50, 50, -50]) self.assertListEqual(decomposed['residual'].values, [0] * 5)
def test_115(self): 'can use .get with default value' obj = TimeSeries(location_id='loc', parameter_id='par') d1 = datetime(1979, 3, 15, 9, 35) obj.add_value(d1, 1.23) # executing __setitem__ ## finds values that are there [self.assertEquals(obj._events[d], obj.get(d)) for d in obj._events.keys()] d2 = datetime(1979, 5, 15, 9, 35) ## returns default value if event is not there self.assertEquals(None, obj.get(d2))
def test_200(self): 'represent empty TimeSeries as Element' obj = TimeSeries(location_id='loc', parameter_id='par') current = obj._as_element() self.assertTrue(isinstance(current, ElementTree.Element)) self.assertEquals('series', current.tag) childElements = [i for i in current.getchildren()] self.assertEquals(1, len(childElements)) self.assertEquals(['header'], [i.tag for i in childElements]) self.assertEquals({}, current.attrib)
def test_022(self): 'start and end of a time series with events' obj = TimeSeries(location_id='loc', parameter_id='par') d1 = datetime(1979, 3, 15, 9, 35) d3 = datetime(1979, 4, 12, 9, 35) d2 = datetime(1979, 5, 15, 9, 35) obj[d1] = 1.23 obj[d3] = 0.23 obj[d2] = -3.01 self.assertEqual(d1, obj.get_start_date()) self.assertEqual(d2, obj.get_end_date())
def test_023(self): 'getting events of a non empty time series' obj = TimeSeries(location_id='loc', parameter_id='par') d1 = datetime(1979, 3, 15, 9, 35) d3 = datetime(1979, 4, 12, 9, 35) d2 = datetime(1979, 5, 15, 9, 35) obj[d1] = 1.23 obj[d3] = 0.23 obj[d2] = -3.01 self.assertEqual(3, len(obj.get_events())) self.assertEqual(2, len(obj.get_events(d3))) self.assertEqual(2, len(obj.get_events(d1, d3)))
def test_201(self): 'represent TimeSeries with two events as Element' obj = TimeSeries(location_id='loc', parameter_id='par') obj[datetime(1980, 11, 23, 19, 35)] = -1 current = obj._as_element() self.assertTrue(isinstance(current, ElementTree.Element)) self.assertEquals('series', current.tag) childElements = [i for i in current.getchildren()] self.assertEquals(2, len(childElements)) self.assertEquals(['header', 'event'], [i.tag for i in childElements]) self.assertEquals({}, current.attrib)
def test010(self): '''result of TimeSeries.as_dict is indexed on locationId parameterId 2-tuples''' obj = TimeSeries.as_dict(self.testdata + "read.PI.timezone.2.xml") self.assertEquals(set([("600", "P1201"), ("600", "P2504")]), set(obj.keys())) self.assertTrue(isinstance(obj[("600", "P1201")], TimeSeries)) self.assertTrue(isinstance(obj[("600", "P2504")], TimeSeries))
def test111(self): 'TimeSeries.as_dict reads events of series (b)' obj = TimeSeries.as_dict(self.testdata + "read.PI.timezone.2.xml") ts = obj[("600", "P2504")] self.assertEquals([ (str_to_datetime("2010-04-05", "00:00:00", 2), 17), (str_to_datetime("2010-04-08", "00:00:00", 2), 22), (str_to_datetime("2010-04-10", "00:00:00", 2), 24), ], ts.get_values())
def test113(self): 'TimeSeries.get_values with only requested timeseries' obj = TimeSeries.as_dict(self.testdata + "read.PI.timezone.2.xml") ts = obj[("600", "P2504")] dates = [str_to_datetime("2010-04-05", "00:00:00", 2), str_to_datetime("2010-04-10", "00:00:00", 2)] self.assertEquals([ (str_to_datetime("2010-04-05", "00:00:00", 2), (17, 0, '')), (str_to_datetime("2010-04-10", "00:00:00", 2), (24, 0, '')), ], ts.get_events(dates=dates))
def test_042(self): 'using deprecated events function' root = logging.getLogger() handler = mock.Handler() root.addHandler(handler) obj = TimeSeries(location_id='loc', parameter_id='par') d1 = datetime(1979, 3, 15, 9, 35) d3 = datetime(1979, 4, 12, 9, 35) d2 = datetime(1979, 5, 15, 9, 35) obj[d1] = 1.23 obj[d3] = 0.23 obj[d2] = -3.01 self.assertEqual(62, len(list(obj.events()))) self.assertEqual(34, len(list(obj.events(d3)))) self.assertEqual(29, len(list(obj.events(d1, d3)))) self.assertEqual(3, len(handler.content)) self.assertEqual("timeseries.timeseries|WARNING|Call to deprecated function events.", handler.content[0]) root.removeHandler(handler)
def write_to_pi_file(*args, **kwargs): """Write the given timeseries in PI XML format. Parameters: *kwargs['filename']* name of PI XML file to create and write to *kwargs['timeseries']* single time series, or a dict of time series, where each time series has with a method 'events' to generate all date, value pairs """ multiple_series_stub = kwargs['timeseries'] if isinstance(multiple_series_stub, dict): multiple_series = [] for parameter_id, series_stub in multiple_series_stub.iteritems(): my_kwargs = deepcopy(kwargs) my_kwargs["parameter_id"] = parameter_id series = TimeSeries(*args, **my_kwargs) series.sorted_event_items = lambda s=series_stub: list(s.events()) multiple_series.append(series) multiple_series.sort(key=lambda series: series.parameter_id) else: series = TimeSeries(*args, **kwargs) series.sorted_event_items = lambda: list(multiple_series_stub.events()) multiple_series = [series] TimeSeries.write_to_pi_file(kwargs['filename'], multiple_series)
def test_kernelcorr(): t1 = TimeSeries([1, 2, 3, 4], [40, 50, 60, 70]) t2 = TimeSeries([1, 2, 3, 4], [40, 50, 60, 70]) standts1 = _corr.stand(t1, t1.mean(), t1.std()) standts2 = _corr.stand(t2, t2.mean(), t2.std()) #Kernel_corr should return a correlation of 1.0 since we use similar timeseries assert(_corr.kernel_corr(standts1, standts2, mult=1) == 1.0)
def test_invalid_moving_average(self): series = TimeSeries([]) with self.assertRaises(ArithmeticError): series.moving_average(3) series = TimeSeries([ (1, 1), (2, 2) ]) with self.assertRaises(ArithmeticError): series.moving_average(3)
def test_procs(): # check that standardization is successful _, t1 = tsmaker(0.5, 0.1, 0.01) # ignore meta-data returned t2 = random_ts(2) standts1 = stand(t1, t1.mean(), t1.std()) standts2 = stand(t2, t2.mean(), t2.std()) assert np.round(standts1.mean(), 10) == 0.0 assert np.round(standts1.std(), 10) == 1.0 assert np.round(standts2.mean(), 10) == 0.0 assert np.round(standts2.std(), 10) == 1.0 # once more, with hard-coded data so we know what answers to expect v1 = [2.00984793, 3.94985729, 0.51427819, 4.16184495, 2.73640138, 0.07386398, 1.32847121, 0.3811719, 4.34006452, 1.86213488] v2 = [6.43496991, 10.34439479, 11.71829468, 3.92319708, 7.07694841, 6.7165553, 5.72293448, 4.79283759, 11.74512723, 11.74048488] t1 = TimeSeries(np.arange(0.0, 1.0, 0.1), v1) t2 = TimeSeries(np.arange(0.0, 1.0, 0.1), v2) standts1 = stand(t1, t1.mean(), t1.std()) standts2 = stand(t2, t2.mean(), t2.std()) assert np.round(standts1.mean(), 10) == 0.0 assert np.round(standts1.std(), 10) == 1.0 assert np.round(standts2.mean(), 10) == 0.0 assert np.round(standts2.std(), 10) == 1.0 idx, mcorr = max_corr_at_phase(standts1, standts2) assert idx == 2 assert np.round(mcorr, 4) == 0.5207 sumcorr = kernel_corr(standts1, standts2, mult=10) assert np.round(sumcorr, 4) == 0.0125
def test_interpolate(self): """ test TimeSeries.interpolate """ tzstr = "US/Eastern" tz = pytz.timezone(tzstr) h = TimeSeries.hour n = datetime.datetime.now() times1 = [n, n+h, n+2*h, n+3*h] times2 = [n, n+3*h] times2 = TimeSeries.makeAware(times2, tz) values = [6.0, 8.0] ts = TimeSeries("test", times2, values, tzstr) res = ts.interpolate(times1) target = [6.0, 6.66667, 7.33333, 8.0] for i in range(len(res)): self.failUnlessAlmostEqual(res[i], target[i], places=4) times3 = [n-h, n+2*h, n+4*h] res = ts.interpolate(times3) target = [None, 7.3333333, None] for i in range(len(res)): self.failUnlessAlmostEqual(res[i], target[i], places=4)
def proc_main(pk, row, arg): ''' Calculates the distance between two time series, using the normalized kernelized cross-correlation. Note: used directly for augmented selects. Parameters ---------- pk : any hashable type The primary key of the database entry row : dictionary The database entry Returns ------- [damean, dastd] : list of floats Mean and standard deviation of the time series data ''' # recast the argument as a time series (type is lost due to serialization) if isinstance(arg, TimeSeries): argts = arg # for server-side testing else: argts = TimeSeries(*arg) # for live client-side operations # standardize the time series stand_argts = stand(argts, argts.mean(), argts.std()) # standardize each row of the data that has tripped the trigger stand_rowts = stand(row['ts'], row['ts'].mean(), row['ts'].std()) # compute the normalized kernelized cross-correlation between the # time series being upserted/selected and the time series argument kerncorr = kernel_corr(stand_rowts, stand_argts, 5) # use the normalized kernelized cross-correlation to compute the distance # between the time series and return return [np.sqrt(2*(1-kerncorr))]
def test_maxcorr(): t1 = TimeSeries([1, 2, 3, 4], [40, 50, 60, 70]) t2 = TimeSeries([1, 2, 3, 4], [50, 60, 70, 40]) standts1 = _corr.stand(t1, t1.mean(), t1.std()) standts2 = _corr.stand(t2, t2.mean(), t2.std()) idx, mcorr = _corr.max_corr_at_phase(standts1, standts2) #idx should be equal to one since the second ts is shifted by 1 assert(idx == 1) assert(np.real(mcorr) == 4)
def test352(self): 'TimeSeries.as_dict filters based on timestamps from django QuerySet' DT = datetime testdata = django.QuerySet([ {'location': '124', 'parameter': 'Q', 'events': [(DT(2011, 11, 11, 12, 20), 0.1, 8, ''), (DT(2011, 11, 11, 12, 25), 0.2, 1, ''), (DT(2011, 11, 11, 12, 30), 0.3, 2, '')]}, ]) end = DT(2011, 11, 11, 12, 25) obj = TimeSeries.as_dict(testdata, end=end) self.assertEquals(set([('124', 'Q')]), set(obj.keys())) self.assertEquals([((), {'timestamp__lte': end})], testdata.filtered)
def test100(self): 'TimeSeries.as_dict reads events of series (a)' obj = TimeSeries.as_dict(self.testdata + "read.PI.timezone.2.xml") ts = obj[("600", "P1201")] self.assertEquals([ (str_to_datetime("2010-04-03", "00:00:00", 2), (20, 0, '')), (str_to_datetime("2010-04-04", "00:00:00", 2), (22, 0, '')), (str_to_datetime("2010-04-05", "00:00:00", 2), (17, 0, '')), (str_to_datetime("2010-04-06", "00:00:00", 2), (20, 0, '')), (str_to_datetime("2010-04-07", "00:00:00", 2), (21, 0, '')), (str_to_datetime("2010-04-08", "00:00:00", 2), (22, 0, '')), (str_to_datetime("2010-04-09", "00:00:00", 2), (24, 0, '')), (str_to_datetime("2010-04-10", "00:00:00", 2), (24, 0, '')), (str_to_datetime("2010-04-11", "00:00:00", 2), (24, 0, '')), (str_to_datetime("2010-04-12", "00:00:00", 2), (22, 0, '')), ], ts.get_events())
def test110(self): 'TimeSeries.as_dict reads events of series (a)' obj = TimeSeries.as_dict(self.testdata + "read.PI.timezone.2.xml") ts = obj[("600", "P1201")] self.assertEquals([ (str_to_datetime("2010-04-03", "00:00:00", 2), 20), (str_to_datetime("2010-04-04", "00:00:00", 2), 22), (str_to_datetime("2010-04-05", "00:00:00", 2), 17), (str_to_datetime("2010-04-06", "00:00:00", 2), 20), (str_to_datetime("2010-04-07", "00:00:00", 2), 21), (str_to_datetime("2010-04-08", "00:00:00", 2), 22), (str_to_datetime("2010-04-09", "00:00:00", 2), 24), (str_to_datetime("2010-04-10", "00:00:00", 2), 24), (str_to_datetime("2010-04-11", "00:00:00", 2), 24), (str_to_datetime("2010-04-12", "00:00:00", 2), 22), ], ts.get_values())
def test_map_return_type(self): series = TimeSeries([ (1, 2), (3, 4), (5, 6) ]) double = series.map(lambda y: y * 2) self.assertTrue(isinstance(double, TimeSeries)) self.assertListEqual([ (1, 4), (3, 8), (5, 12) ], double.points)
def tsmaker(m, s, j): t = np.arange(0.0, 1.0, 0.01) v = norm.pdf(t, m, s) + j * np.random.randn(100) return ts.TimeSeries(list(v), list(t))
def random_ts(a): t = np.arange(0.0, 1.0, 0.01) v = a * np.random.random(100) return ts.TimeSeries(list(v), list(t))
# Read until the horizon actualData = rawData[rawData.shape[0] - horizon:, 4] rawData = rawData[:rawData.shape[0] - horizon, :] # Normalize the raw data minMax = pp.MinMaxScaler((-1, +1)) rawData[:, 4] = minMax.fit_transform(rawData[:, 4]) # Available data - to store known and unknown availableData = np.zeros((rawData.shape[0] + horizon, 4)) availableData[:rawData.shape[0], 0:3] = rawData[:, 0:3] availableData[:rawData.shape[0], 3] = rawData[:, valueIndex] #Pre-Process the data to form feature and target vectors tsp = ts.TimeSeriesProcessor(rawData[:, valueIndex], depth, 1) processedData = tsp.getProcessedData() #Append bias processedData = np.hstack((np.ones( (processedData.shape[0], 1)), processedData)) #Training data trainingInputData = processedData[:, :1 + depth] trainingOutputData = processedData[:, 1 + depth:] #Validation data validationInputData = trainingInputData validationOutputData = trainingOutputData #Tune the reservoir
from timeseries import TimeSeries threes = TimeSeries(range(0,1000,3)) fives = TimeSeries(range(0,1000,5)) s = 0 for i in range(0,1000): if i in threes or i in fives: s += i print("sum",s)
event_list = np.concatenate((event_list, tmp)).flatten() # Event list np.random.shuffle(event_list) # Poisson mean time between events lam = 1.0 print 'Number of events %i' % (len(event_list)) # t = 0.01 * np.arange(lam * event_list.size) tmin = np.min(t) tmax = np.max(t) emission = np.zeros_like(t) start_time = 0.0 for ev in event_list: T = (1.0 * ev)**(-power) energy = T**(1.0 + gamma) start_time = np.random.uniform(low=tmin, high=tmax) emnew = event(t, start_time, energy, T) if np.max(emnew) >= 0.0: emission = emission + emnew else: print('Not big enough') ts = TimeSeries(t[::10], emission[::10]) ts.peek_ps() plt.loglog() plt.show()
def test_invalid_trend(self): series = TimeSeries([]) with self.assertRaises(ArithmeticError): series.trend()
def test_indexing(self): series = TimeSeries([ (1, 3), (2, 3), (3, 3) ]) self.assertEquals(series[1], 3) self.assertEquals(series[2], 3) with self.assertRaises(KeyError): foo = series[4]
'index': 1 }, 'vp': { 'type': "bool", 'index': 1 } } orders = [0, 3, 1, 2] blargs = [1, 1, 2, 2] times = [0, 1, 2, 3, 4] # Same time basis values1 = [0, 2, 4, 6, 8] # Two example time series values values2 = [2, 4, 6, 8, 10] vps = [True, False, False, True] # Vantage points for first and last timeseries tsrs = [TimeSeries(times, values1 if i < 2 else values2) for i in range(4)] # only two value ranges def setup_module(module): if os.path.exists("documents/"): shutil.rmtree('documents/') # Extend schema for i in range(4): if vps[i]: schema["d_vp-{}".format(i)] = {'type': "float", 'index': 1} # Make db db = DocDB('pk', schema)
def __init__(self, schema, pkfield, load=False, dbname="db", overwrite=False, dist=procs.corr_indb, threshold = 10, wordlength = 16, tslen = 256, cardinality = 64): """ Parameters ---------- schema : dict Key = name of field (e.g. 'ts', 'mean') Value = dict of that field's properties. Recognized keys include: 'type': Required for all fields except ts. pkfield must have type str. 'index': Required for all fields. pkfield : str The name of the field which will be the primary key. Must match a key in schema. load : bool Whether to populate the database with an existing one on file. dbname : str Database filename overwrite : bool If load=False, whether to overwrite an existing database. dist : function Calculates the distance between two TimeSeries objects, must take arguments (ts1, ts2) Attributes ---------- indexes : dict Key = fieldname Value = binary search tree (if int or float) or dictionary of sets (otherwise) mapping values to pks rows : dict Key = primary key Value = dict of the fields associated with each key schema : dict (See above) pkfield : str (See above) dbname : str (See above) tslen : int The length of each timeseries in the database, strictly enforced """ # ---- Validating input ---- # if not isinstance(pkfield, str): raise ValueError("Field name must be of type str") if not isinstance(threshold, int): raise ValueError("Threshold must be of type int") if not isinstance(wordlength, int): raise ValueError("Word length must be of type int") if threshold <= 0: raise ValueError("Threshold must be greater than zero") if wordlength <= 0: raise ValueError("Word length must be greater than zero") if '1' in '{0:b}'.format(wordlength)[1:]: raise ValueError("Word length must be a power of two") if not isinstance(tslen, int): raise ValueError("TimeSeries length must be of type int") if tslen < wordlength: raise ValueError("TimeSeries length must be greater than or equal to the word length") if '1' in '{0:b}'.format(tslen)[1:]: raise ValueError("TimeSeries length must be a power of two") if not isinstance(cardinality, int): raise ValueError("Cardinality must be of type int") if cardinality <= 0: raise ValueError("Cardinality must be greater than zero") if '1' in '{0:b}'.format(cardinality)[1:]: raise ValueError("Cardinality must be a power of two") if cardinality > 64: raise ValueError("Cardinalities greater than 64 are not supported") if not isinstance(load, bool): raise ValueError("Load must be of type bool") if not isinstance(dbname, str): raise ValueError("Database name must be string") if not isinstance(overwrite, bool): raise ValueError("Overwrite must be of type bool") if isinstance(schema, dict): for field in schema: if field == 'DELETE': raise ValueError("The fieldname 'DELETE' is forbidden") if ':' in field: raise ValueError("Field names may not contain the ':' character") if field != 'ts': if 'type' not in schema[field]: raise ValueError("Schema must specify type for each non-ts field") if field == pkfield and schema[field]['type'] != str: raise ValueError("Primary key must be of type str") if schema[field]['type'] not in [int, float, bool, str]: raise ValueError("Only types int, float, bool, and str are supported") if field[:5] == 'd_vp-': raise ValueError("Field names beginning with 'd_vp-' are forbidden") if field == 'vp' and schema[field]['type'] != bool: raise ValueError("Field 'vp' must be of boolean type") else: raise ValueError("Schema must be a dictionary") if pkfield not in schema: raise ValueError("Primary key field must be included in schema") # Assign attributes according to schema self.indexes = {} self.rows = {} self.rows_SAX = {} self.wordlength = wordlength self.threshold = threshold self.SAX_tree = Tree_Initializer(threshold = threshold, wordlength = wordlength).tree self.card = cardinality self.schema = schema self.dbname = dbname self.pkfield = pkfield self.tslen = None self.tslen_SAX = tslen self.overwrite = overwrite self.dist = dist self.vps = [] for s in schema: indexinfo = schema[s]['index'] if indexinfo is not None: if schema[s]['type'] == int or schema[s]['type'] == float: self.indexes[s] = BinarySearchTree() else: # Add a bitmask option for strings? self.indexes[s] = defaultdict(set) if load: try: fd = open(dbname) for l in fd.readlines(): [pk, field, val] = l.strip().split(":") if field in self.schema: if pk not in self.rows: self.rows[pk] = {pkfield:pk} else: if self.schema[field]['type'] == bool: if val == 'False': self.rows[pk][field] = False else: self.rows[pk][field] = True else: self.rows[pk][field] = self.schema[field]['type'](val) if pk not in self.rows_SAX: self.rows_SAX[pk] = {pkfield:pk} else: if self.schema[field]['type'] == bool: if val == 'False': self.rows_SAX[pk][field] = False else: self.rows_SAX[pk][field] = True else: self.rows_SAX[pk][field] = self.schema[field]['type'](val) if field == 'vp' and val == 'True': self.vps.append(pk) self.indexes['d_vp-'+pk] = BinarySearchTree() elif field == 'DELETE': if 'vp' in schema and self.rows[pk]['vp'] == True: self.del_vp(pk) del self.rows[pk] del self.rows_SAX[pk] elif field[:5] == 'd_vp-': self.rows[pk][field] = float(val) else: raise IOError("Database is incompatible with input schema") fd.close() # Read in timeseries of non-deleted keys for pk in self.rows: tsarray = np.load(self.dbname+"_ts/"+pk+"_ts.npy") self.rows[pk]['ts'] = TimeSeries(tsarray[0,:], tsarray[1,:]) self.tslen = tsarray.shape[1] #tsarray2 = np.load(self.dbname+"_ts_SAX/"+pk+"_ts_SAX.npy") x1 = np.linspace(min(tsarray[0,:]),max(tsarray[0,:]), self.tslen_SAX) ts_SAX_data = interp1d(tsarray[0,:], tsarray[1,:])(x1) ts_SAX_time = x1 ts_SAX = TimeSeries(ts_SAX_time,ts_SAX_data) self.rows_SAX[pk]['ts'] = ts_SAX rep = isax_indb(ts_SAX,self.card,self.wordlength) self.SAX_tree.insert(pk, rep) self.index_bulk(list(self.rows.keys())) except: raise IOError("Database does not exist or has been corrupted") else: if os.path.exists(dbname) and overwrite == False: raise ValueError("Database of that name already exists. Delete existing db, rename, or set overwrite=True.")
plt.grid(True) plt.show() if __name__ == "__main__": events = queue.Queue() # 同期キュー status = dict() # tick をまたいで記憶しておきたい情報 status["is_sim"] = True portfolio = PortfolioLocal(status) execution = SimulatedExecutionHandler(status) timeseries = TimeSeries(status) strategy = OLSTIME(status) # strategy = SMAKNN(status) # strategy = MARTINRSI(status) # strategy = SMABOLPIPRSI(status) # strategy = SMABOLPIP(status) # strategy = SMAPIP(status) # strategy = SMAPIPRSI(status) # strategy = SMAOLSPIP(status) # strategy = SMABOL(status) # strategy = SMARSIOLS(status) # strategy = WMA(status) # strategy = SMAOLS(status) # strategy = SMA(status) # strategy = RSI(status)
def _load_ts(self, pk): filepath = 'documents/ts/' + pk + '.json' with open(filepath, 'r+') as f: time_series = json.load(f) time_series['ts'] = TimeSeries(*time_series['ts']) return time_series
def get_data(self, channels, start_time, end_time, buffer_time=0.0, resampled_rate=None, filt_freq=None, filt_type='stop', filt_order=4, keep_buffer=False, esrc='esrc', eoffset='eoffset', loop_axis=None, num_mp_procs=0, eoffset_in_time=True): """ Return the requested range of data for each event by using the proper data retrieval mechanism for each event. Parameters ---------- channels: {list,int,None} Channels from which to load data. start_time: {float} Start of epoch to retrieve (in time-unit of the data). end_time: {float} End of epoch to retrieve (in time-unit of the data). buffer_time: {float},optional Extra buffer to add on either side of the event in order to avoid edge effects when filtering (in time unit of the data). resampled_rate: {float},optional New samplerate to resample the data to after loading. filt_freq: {array_like},optional The range of frequencies to filter (depends on the filter type.) filt_type = {scipy.signal.band_dict.keys()},optional Filter type. filt_order = {int},optional The order of the filter. keep_buffer: {boolean},optional Whether to keep the buffer when returning the data. esrc : {string},optional Name for the field containing the source for the time series data corresponding to the event. eoffset: {string},optional Name for the field containing the offset (in seconds) for the event within the specified source. eoffset_in_time: {boolean},optional If True, the unit of the event offsets is taken to be time (unit of the data), otherwise samples. Returns ------- A TimeSeries instance with dimensions (channels,events,time). """ # check for necessary fields if not (esrc in self.dtype.names and eoffset in self.dtype.names): raise ValueError(esrc + ' and ' + eoffset + ' must be valid fieldnames ' + 'specifying source and offset for the data.') # get ready to load dat eventdata = [] events = [] # speed up by getting unique event sources first usources = np.unique(self[esrc]) # loop over unique sources eventdata = None for src in usources: # get the eventOffsets from that source ind = np.atleast_1d(self[esrc] == src) if len(ind) == 1: event_offsets = self[eoffset] events.append(self) else: event_offsets = self[ind][eoffset] events.append(self[ind]) #print "Loading %d events from %s" % (ind.sum(),src) # get the timeseries for those events newdat = src.get_event_data(channels, event_offsets, start_time, end_time, buffer_time, resampled_rate, filt_freq, filt_type, filt_order, keep_buffer, loop_axis, num_mp_procs, eoffset, eoffset_in_time) if eventdata is None: eventdata = newdat else: eventdata = eventdata.extend(newdat, axis=1) # concatenate (must eventually check that dims match) tdim = eventdata['time'] cdim = eventdata['channels'] srate = eventdata.samplerate events = np.concatenate(events).view(self.__class__) eventdata = TimeSeries(eventdata, 'time', srate, dims=[cdim, Dim(events, 'events'), tdim]) return eventdata
def score_all(functionNode): """ score all thresholds again by using the stream implementation #works only on context of the class object """ logger = functionNode.get_logger() logger.debug("score_all") progressNode = functionNode.get_child("control").get_child("progress") progressNode.set_value(0) model = functionNode.get_model() # for the model API annos = functionNode.get_child("annotations").get_leaves() annos = [ anno for anno in annos if anno.get_child("type").get_value() == "time" ] #only the time annotations variableIds = functionNode.get_child( "variables").get_leaves_ids() # the variableids to work on try: overWrite = functionNode.get_child("overWrite").get_value() except: overWrite = True obj = functionNode.get_parent().get_object() obj.reset( ) #read the new thresholds into the object!! this also affects parallel streaming processes # for each id (variable) that has threshold(s) # take the values and times of that varialbe # find out the annotations we need, create the stream data blob, send it over progressStep = 1 / float(len(obj.get_thresholds())) total = None for id, thresholdsInfo in obj.get_thresholds().items( ): # thresholds is a dict of {id: {tag:{"min":0,"max":1}, tag2:{} .. ,id2:{}} if id not in variableIds: continue # skip this one, is not selected progressNode.set_value(progressNode.get_value() + progressStep) var = model.get_node(id) data = var.get_time_series() times = data["__time"] #now produce the interesting states blob = { "type": "timeseries", "data": { "__time": times, id: data["values"], "__states": {} } } for state in thresholdsInfo.keys( ): #iterate over the states where the variable has special thresholds myAnnos = mh.filter_annotations(annos, state) stateMask = mh.annotations_to_class_vector(myAnnos, data["__time"]) stateMask = numpy.isfinite(stateMask) blob["data"]["__states"][state] = stateMask #now we have prepared a data and state blob, we will now score by feeding it into the stream scorer #del blob["data"]["__states"]#for test, now blob = obj.feed(blob) #now the blob contains more entries, e.g. the score variable id and the according scores, that is what we want for blobId, values in blob["data"].items(): if blobId not in ["__time", id, "__states"]: #this is the score, overwrite the whole thing scoreNode = model.get_node(blobId) if scoreNode.get_name() == "_total_score": continue # this is the combined result of several variables going into the stream scoring, not relevant here scoreNode.set_time_series( values=values, times=times ) # xxx is set ok here, or do we need "insert" to make sure there has not been changed in the meantime? model.notify_observers(scoreNode.get_parent().get_id(), "children") # we trigger # build the total score: # merge in the new times, resample the total score, resampel the local score, then merge them # the merge function will use the new values whereever there is one (empty fields are named "nan" # for the total score, we need a resampling to avoid the mixing of results e.g. # two sensor have different result during a given interval, but different times, if we just merge # we get a True, False, True,False mixture # so we build the merge vector, first resample then merge values[numpy.isfinite( values)] = -1 # set -1 for all out of limit if type(total) is type(None): total = TimeSeries(values=values, times=times) else: local = TimeSeries(values=values, times=times) total.merge( local ) # the merge resamples the incoming data to the existing time series, NaN will be replaced by new values, # finally, write the total # if the overWrite is True, we replace, otherwise we merge with the existing, previous result totalScoreNode = functionNode.get_parent().get_child("output").get_child( "_total_score") if overWrite: totalScoreNode.set_time_series(values=total.get_values(), times=total.get_times()) else: totalScoreNode.merge_time_series(values=total.get_values(), times=total.get_times()) return True
app = Flask(__name__) database_info = { "url": "localhost", "port": "27017", "database": "Sibyl" } label = "analyzer" db_manager = DatabaseManager(database_info, label) global pewma global cl global ts cl = StaticControlLimits() pewma_model = Pewma() ts = TimeSeries() @app.route('/static_control_limits', methods=['POST']) def static_control_limits(): """ Function used to check if data is above set threshold Args: data (dict): the raw data Returns: dict """ try: content = request.get_json() try:
def test_iteration(self): points = [ (1, 2), (3, 4), (5, 6) ] series = TimeSeries(points) self.assertListEqual([ s for s in series ], points)
# Create a time series object dt = 12.0 t = dt * np.arange(0, nt) # Fix the input datacube for any non-finite data for i in range(0, nx): for j in range(0, ny): d = dc[j, i, :].flatten() # Fix the data for any non-finite entries d = tsutils.fix_nonfinite(d) # Remove the mean d = d - np.mean(d) dc[j, i, :] = d # Time series datacube dcts = TimeSeries(t, dc) dcts.name = 'AIA ' + str(wave) + '-' + stype + ': ' + location # Get the Fourier power pwr = dcts.ppower # Arithmetic mean of the Fourier power iobs = np.mean(pwr, axis=(0, 1)) # Sigma for the fit to the power sigma = np.std(pwr, axis=(0, 1)) # Result # 1 - add up all the emission and do the analysis on the full FOV # Also, make a histogram of all the power spectra to get an idea of the # varition present
power_noise=False) # Noise noise_pls = SimplePowerLawSpectrum([10.0, 2.0], nt=nt, dt=dt) rn = TimeSeriesFromPowerSpectrum(noise_pls, V=V, W=W) noise = 0.0 * rn.sample # Create the simulated data amplitude = 100 data = amplitude * tsnew.sample + noise # Sample times t = dt * np.arange(0, nt) # Time series object ts = TimeSeries(t, data) # Scaled frequency freqs = ts.PowerSpectrum.frequencies.positive / ts.PowerSpectrum.frequencies.positive[ 0] # Form the input for the MCMC algorithm. this = ([ts.pfreq, ts.ppower], ) norm_estimate = np.zeros((3, )) norm_estimate[0] = ts.ppower[0] norm_estimate[1] = norm_estimate[0] / 1000.0 norm_estimate[2] = norm_estimate[0] * 1000.0 background_estimate = np.zeros_like(norm_estimate) background_estimate[0] = np.mean(ts.ppower[-10:-1])
wave, location='/Users/ireland/Data/AIA_Data/shutdownfun/' + choice, derotate=False) #wave = '171' #dc, location = aia_specific.rn4(wave, Xrange=[-201, -1]) # Get some properties of the datacube ny = dc.shape[0] nx = dc.shape[1] nt = dc.shape[2] # Create a time series object dt = 12.0 t = dt * np.arange(0, nt) tsdummy = TimeSeries(t, t) iobs = np.zeros(tsdummy.PowerSpectrum.Npower.shape) logiobs = np.zeros(tsdummy.PowerSpectrum.Npower.shape) nposfreq = len(iobs) # Result # 1 - add up all the emission and do the analysis on the full FOV # Also, make a histogram of all the power spectra to get an idea of the # varition present # Sum over all the spatial locations full_data = sum_over_space(dc) # #full_data = tsutils.fix_nonfinite(dc[10, 10, :]) # Average emission over all the data
def test_stand(): t1 = TimeSeries([1, 2, 3, 4], [40, 50, 60, 70]) val = _corr.stand(np.array(t1.values()), 55.0, 10) assert (list(val) == [-1.5, -0.5, 0.5, 1.5])
def get_event_data(self, channels, events, start_time, end_time, buffer_time=0.0, resampled_rate=None, filt_freq=None, filt_type='stop', filt_order=4, keep_buffer=False, loop_axis=None, num_mp_procs=0, eoffset='eoffset', eoffset_in_time=True): """ Return an TimeSeries containing data for the specified channel in the form [events,duration]. Parameters ---------- channels: {int} or {dict} Channels from which to load data. events: {array_like} or {recarray} Array/list of event offsets (in time or samples as specified by eoffset_in_time; in time by default) into the data, specifying each event onset time. start_time: {float} Start of epoch to retrieve (in time-unit of the data). end_time: {float} End of epoch to retrieve (in time-unit of the data). buffer_time: {float},optional Extra buffer to add on either side of the event in order to avoid edge effects when filtering (in time unit of the data). resampled_rate: {float},optional New samplerate to resample the data to after loading. filt_freq: {array_like},optional The range of frequencies to filter (depends on the filter type.) filt_type = {scipy.signal.band_dict.keys()},optional Filter type. filt_order = {int},optional The order of the filter. keep_buffer: {boolean},optional Whether to keep the buffer when returning the data. eoffset_in_time: {boolean},optional If True, the unit of the event offsets is taken to be time (unit of the data), otherwise samples. """ # translate back to dur and offset dur = end_time - start_time offset = start_time buf = buffer_time # get the event offsets if ((not (hasattr(events, 'dtype') or hasattr(events, 'columns'))) or (hasattr(events, 'dtype') and events.dtype.names is None)): # they just passed in a list event_offsets = events elif ((hasattr(events, 'dtype') and (eoffset in events.dtype.names)) or (hasattr(events, 'columns') and (eoffset in events.columns))): event_offsets = events[eoffset] else: raise ValueError(eoffset + ' must be a valid fieldname ' + 'specifying the offset for the data.') # Sanity checks: if (dur < 0): raise ValueError('Duration must not be negative! ' + 'Specified duration: ' + str(dur)) if (np.min(event_offsets) < 0): raise ValueError('Event offsets must not be negative!') # make sure the events are an actual array: event_offsets = np.asarray(event_offsets) if eoffset_in_time: # convert to samples event_offsets = np.atleast_1d( np.int64(np.round(event_offsets * self.samplerate))) # set event durations from rate # get the samplesize samplesize = 1. / self.samplerate # get the number of buffer samples buf_samp = int(np.ceil(buf / samplesize)) # calculate the offset samples that contains the desired offset offset_samp = int( np.ceil((np.abs(offset) - samplesize * .5) / samplesize) * np.sign(offset)) # finally get the duration necessary to cover the desired span #dur_samp = int(np.ceil((dur - samplesize*.5)/samplesize)) dur_samp = (int(np.ceil( (dur + offset - samplesize * .5) / samplesize)) - offset_samp + 1) # add in the buffer dur_samp += 2 * buf_samp offset_samp -= buf_samp # check that we have all the data we need before every event: if (np.min(event_offsets + offset_samp) < 0): bad_evs = ((event_offsets + offset_samp) < 0) raise ValueError('The specified values for offset and buffer ' + 'require more data than is available before ' + str(np.sum(bad_evs)) + ' of all ' + str(len(bad_evs)) + ' events.') # process the channels if isinstance(channels, dict): # turn into indices ch_info = self.channels key = channels.keys()[0] channels = [ np.nonzero(ch_info[key] == c)[0][0] for c in channels[key] ] elif isinstance(channels, str): # find that channel by name channels = np.nonzero(self.channels['name'] == channels)[0][0] if channels is None or len(np.atleast_1d(channels)) == 0: channels = np.arange(self.nchannels) channels = np.atleast_1d(channels) channels.sort() # load the timeseries (this must be implemented by subclasses) eventdata = self._load_data(channels, event_offsets, dur_samp, offset_samp) # calc the time range # get the samplesize samp_start = offset_samp * samplesize samp_end = samp_start + (dur_samp - 1) * samplesize time_range = np.linspace(samp_start, samp_end, dur_samp) # when channels is an array of channels labels i.e. strings like '002','003',... # we need to use xray arrays to do fancy indexing if channels.dtype.char == 'S': try: # perhaps we should vectorize it... selector_array = [ np.where(self.channels.name == channel)[0][0] for channel in channels ] selected_channels = self.channels[selector_array] # from xray import DataArray # self.channels_xray = DataArray(self.channels.number,coords=[self.channels.name],dims=['name']) # self.channels_xray = self.channels_xray.loc[channels] # # self.channels_xray=np.rec.fromarrays([self.channels_xray.values,self.channels_xray.coords['name'].values],names='number,name') except ImportError: pass dims = [ Dim(selected_channels, 'channels'), # can index into channels Dim(events, 'events'), Dim(time_range, 'time') ] # dims = [Dim(self.channels_xray,'channels'), # can index into channels # Dim(events,'events'), # Dim(time_range,'time')] else: # make it a timeseries # ORIGINAL CODE dims = [ Dim(self.channels[channels], 'channels'), # can index into channels Dim(events, 'events'), Dim(time_range, 'time') ] # # make it a timeseries # dims = [Dim(self.channels[channels],'channels'), # can index into channels # Dim(events,'events'), # Dim(time_range,'time')] eventdata = TimeSeries(np.asarray(eventdata), 'time', self.samplerate, dims=dims) # filter if desired if not (filt_freq is None): # filter that data eventdata = eventdata.filtered(filt_freq, filt_type=filt_type, order=filt_order) # resample if desired if (not (resampled_rate is None) and not (resampled_rate == eventdata.samplerate)): # resample the data eventdata = eventdata.resampled(resampled_rate, loop_axis=loop_axis, num_mp_procs=num_mp_procs) # remove the buffer and set the time range if buf > 0 and not (keep_buffer): # remove the buffer eventdata = eventdata.remove_buffer(buf) # return the timeseries return eventdata
def test_delete(): client.insert_ts('test', TimeSeries(times, values1)) client.delete_ts('test')
red_noise = False dt = 12.0 nt = 300 np.random.seed(seed=1) model_param = [10.0, 1.77, -100.0] pls1 = SimplePowerLawSpectrumWithConstantBackground(model_param, nt=nt, dt=dt) data = TimeSeriesFromPowerSpectrum(pls1).sample t = dt * np.arange(0, nt) amplitude = 0.0 data = data + amplitude * (data.max() - data.min()) * np.sin(2 * np.pi * t / 300.0) # Create a time series object ts = TimeSeries(t, data) ts.label = 'emission' ts.units = 'arb. units' ts.name = 'simulated data [n=%4.2f]' % (model_param[1]) # Get the normalized power and the positive frequencies iobs = ts.PowerSpectrum.ppower this = ([ts.PowerSpectrum.frequencies.positive, iobs],) # _____________________________________________________________________________ # ----------------------------------------------------------------------------- # Wavelet transform using a white noise background # ----------------------------------------------------------------------------- var = ts.data # Range of periods to average avg1, avg2 = (150.0, 400.0)
dt = 12.0 nt = 1800 period = 300.0 window = 31 np.random.seed(seed=1) t = dt * np.arange(0, nt) noise = np.random.normal(size=nt) amplitude = 1.0 data1 = amplitude * np.sin(2 * np.pi * t / period) + noise + 10 data2 = tsutils.movingaverage(data1, window) ts1 = TimeSeries(t, data1) ts2 = TimeSeries(t, data2) ts3 = TimeSeries(t, data1 - data2) plt.figure(1) plt.plot(t, data1, label='original time series') plt.plot(t, data2, label='moving average') plt.plot(t, data1 - data2, label='original - moving average') plt.xlabel('time') plt.ylabel('emission (arbitrary units)') plt.legend() window = window / 2 w1 = 1.0 / (2 * window + 1)
def stand(x, m, s): "standardize timeseries x my mean m and std deviation s" return ts.TimeSeries(list(x.values() - m / s), list(x.times()))
from performance import ErrorMetrics as rmse from datetime import datetime import numpy as np import os from datetime import date, timedelta from timeseries import TimeSeries as ts # Read data from the file rawData = np.loadtxt("facebookFansHistory_bmw_raw.txt", delimiter=',') data = rawData[:rawData.shape[0], rawData.shape[1] - 1].reshape( (rawData.shape[0], 1)) depth = 1 horizon = 1 tsp = ts.TimeSeriesProcessor(rawData, depth, horizon, 4) processedData = tsp.getProcessedData() inputData = np.hstack((np.ones((processedData.shape[0], 1)), processedData[:processedData.shape[0], :depth])) outputData = processedData[:processedData.shape[0], depth:depth + horizon] # Train inputWeightRandom = np.load("Outputs/inputWeight.npy") reservoirWeightRandom = np.load("Outputs/reservoirWeight.npy") res = reservoir.Reservoir(size=600, spectralRadius=1.25, inputScaling=0.50, reservoirScaling=0.50, leakingRate=0.20, initialTransient=0,
nt = 300 V = 100 W = 100 no_noise = False pls = SimplePowerLawSpectrum([10.0, 2.0], nt=nt, dt=dt) tsnew3 = TimeSeriesFromPowerSpectrum(pls, V=V, W=W) i = 0 ncount = 10000 data3 = np.zeros(shape=(nt)) while i < ncount: i = i + 1 data3 = data3 + tsnew3.sample * np.random.pareto(2.0) ts3 = TimeSeries(dt * np.arange(0, nt), data3) this = ([ts3.pfreq, ts3.ppower], ) norm_estimate = np.zeros((3, )) norm_estimate[0] = ts3.ppower[0] norm_estimate[1] = norm_estimate[0] / 1000.0 norm_estimate[2] = norm_estimate[0] * 1000.0 background_estimate = np.zeros_like(norm_estimate) background_estimate[0] = np.mean(ts3.ppower[-10:-1]) background_estimate[1] = background_estimate[0] / 1000.0 background_estimate[2] = background_estimate[0] * 1000.0 estimate = { "norm_estimate": norm_estimate,
def insert_ts(self, pk, ts): try: pk = str(pk) except: raise ValueError("Primary keys must be string-compatible") if ':' in pk: raise ValueError("Primary keys may not include the ':' character") if not isinstance(ts, TimeSeries): raise ValueError('Must insert a TimeSeries object') if pk not in self.rows: self.rows[pk] = {self.pkfield:pk} else: raise ValueError('Duplicate primary key found during insert') if pk not in self.rows_SAX: self.rows_SAX[pk] = {self.pkfield:pk} else: raise ValueError('Duplicate primary key found during insert') # Save timeseries as a 2d numpy array if self.tslen is None: self.tslen = len(ts) elif len(ts) != self.tslen: raise ValueError('All timeseries must be of same length') if not os.path.exists(self.dbname+"_ts"): os.makedirs(self.dbname+"_ts") np.save(self.dbname+"_ts/"+pk+"_ts.npy", np.vstack((ts.time, ts.data))) x1 = np.linspace(min(ts.time),max(ts.time), self.tslen_SAX) ts_SAX_data = interp1d(ts.time, ts.data)(x1) ts_SAX_time = x1 ts_SAX = TimeSeries(ts_SAX_time,ts_SAX_data) if not os.path.exists(self.dbname+"_ts_SAX"): os.makedirs(self.dbname+"_ts_SAX") np.save(self.dbname+"_ts_SAX/"+pk+"_ts_SAX.npy", np.vstack((ts_SAX.time, ts_SAX.data))) # Save a record in the database file if self.overwrite or not os.path.exists(self.dbname): fd = open(self.dbname, 'w') self.overwrite = False else: fd = open(self.dbname, 'a') fd.write(pk+':'+self.pkfield+':'+pk+'\n') if 'vp' in self.schema: fd.write(pk+':vp:False\n') fd.close() self.rows[pk]['ts'] = ts if 'vp' in self.schema: self.rows[pk]['vp'] = False self.rows_SAX[pk]['ts'] = ts_SAX rep = isax_indb(ts_SAX,self.card,self.wordlength) self.SAX_tree.insert(pk, rep) if 'vp' in self.schema: self.rows_SAX[pk]['vp'] = False for vp in self.vps: ts1 = self.rows[vp]['ts'] self.upsert_meta(pk, {'d_vp-'+vp : self.dist(ts1,ts)}) self.update_indices(pk)
# Create a location to save the figures savefig = os.path.join(os.path.expanduser(savefig), wave) if not (os.path.isdir(savefig)): os.makedirs(savefig) figname = data_name savefig = os.path.join(savefig, figname) # Get some properties of the datacube ny = dc.shape[0] nx = dc.shape[1] nt = dc.shape[2] # Create a time series object dt = 12.0 t = dt * np.arange(0, nt) tsdummy = TimeSeries(t, t) iobs = np.zeros(tsdummy.PowerSpectrum.Npower.shape) logiobs = np.zeros(tsdummy.PowerSpectrum.Npower.shape) nposfreq = len(iobs) # Result # 1 - add up all the emission and do the analysis on the full FOV # Also, make a histogram of all the power spectra to get an idea of the # varition present # storage pwr = np.zeros((ny, nx, nposfreq)) logpwr = np.zeros_like(pwr) full_data = np.zeros((nt)) for i in range(0, nx): for j in range(0, ny): d = dc[j, i, :].flatten()
def get_event_data(self, channels, event_offsets, start_time, end_time, buffer_time=0.0, resampled_rate=None, filt_freq=None, filt_type='stop', filt_order=4, keep_buffer=False): """ Return an TimeSeries containing data for the specified channel in the form [events,duration]. Parameters ---------- channels: {int} Channels from which to load data. event_offsets: {array_like} Array/list of event offsets (in samples) into the data, specifying each event onset time. start_time: {float} Start of epoch to retrieve (in time-unit of the data). end_time: {float} End of epoch to retrieve (in time-unit of the data). buffer_time: {float},optional Extra buffer to add on either side of the event in order to avoid edge effects when filtering (in time unit of the data). resampled_rate: {float},optional New samplerate to resample the data to after loading. filt_freq: {array_like},optional The range of frequencies to filter (depends on the filter type.) filt_type = {scipy.signal.band_dict.keys()},optional Filter type. filt_order = {int},optional The order of the filter. keep_buffer: {boolean},optional Whether to keep the buffer when returning the data. """ # translate back to dur and offset dur = end_time - start_time offset = start_time buf = buffer_time # Sanity checks: if (dur < 0): raise ValueError('Duration must not be negative! ' + 'Specified duration: ' + str(dur)) if (np.min(event_offsets) < 0): raise ValueError('Event offsets must not be negative!') # make sure the events are an actual array event_offsets = np.asarray(event_offsets) # set event durations from rate # get the samplesize samplesize = 1. / self.samplerate # get the number of buffer samples buf_samp = int(np.ceil(buf / samplesize)) # calculate the offset samples that contains the desired offset offset_samp = int( np.ceil((np.abs(offset) - samplesize * .5) / samplesize) * np.sign(offset)) # finally get the duration necessary to cover the desired span #dur_samp = int(np.ceil((dur - samplesize*.5)/samplesize)) dur_samp = (int(np.ceil( (dur + offset - samplesize * .5) / samplesize)) - offset_samp + 1) # add in the buffer dur_samp += 2 * buf_samp offset_samp -= buf_samp # check that we have all the data we need before every event: if (np.min(event_offsets + offset_samp) < 0): bad_evs = ((event_offsets + offset_samp) < 0) raise ValueError('The specified values for offset and buffer ' + 'require more data than is available before ' + str(np.sum(bad_evs)) + ' of all ' + str(len(bad_evs)) + ' events.') # process the channels if channels is None or len(np.atleast_1d(channels)) == 0: channels = np.arange(self.nchannels) channels = np.atleast_1d(channels) # load the timeseries (this must be implemented by subclasses) eventdata = self._load_data(channels, event_offsets, dur_samp, offset_samp) # calc the time range # get the samplesize samp_start = offset_samp * samplesize samp_end = samp_start + (dur_samp - 1) * samplesize time_range = np.linspace(samp_start, samp_end, dur_samp) # make it a timeseries dims = [ Dim(channels, 'channels'), Dim(event_offsets, 'event_offsets'), Dim(time_range, 'time') ] eventdata = TimeSeries(np.asarray(eventdata), 'time', self.samplerate, dims=dims) # filter if desired if not (filt_freq is None): # filter that data eventdata = eventdata.filtered(filt_freq, filt_type=filt_type, order=filt_order) # resample if desired if (not (resampled_rate is None) and not (resampled_rate == eventdata.samplerate)): # resample the data eventdata = eventdata.resampled(resampled_rate) # remove the buffer and set the time range if buf > 0 and not (keep_buffer): # remove the buffer eventdata = eventdata.remove_buffer(buf) # return the timeseries return eventdata