def flatten(ts, components=True, depth=np.inf): """ Create a :py:class:`TimeSeriesCollection` with passed time series and their components. Parameters ---------- ts: type convertible to :py:obj:`TimeSeriesCollection` Time series collection whose components are to be "flattened". components: list(str), optional Components to include in the final time series collection. Default is all components. depth: int, optional Depth of components to include in the final time series collection. Default is infinity. Returns ------- :py:obj:`TimeSeriesCollection` Flattened time series collection. """ if isinstance(components, list): components = [key.upper() for key in components] flattened = TimeSeriesCollection() temp_ts_collection = convert_to_ts_collection(ts) counter = 0 while counter < depth: ts_collection = temp_ts_collection if not ts_collection: break flattened.add_list(ts_collection) temp_ts_collection = TimeSeriesCollection() counter += 1 for ts in ts_collection: instantiate_components(ts, components, temp_ts_collection) return flattened
def write_attributes(self, ts_collection, components=True, depth=np.inf): """ Write time series attributes to the database. Parameters ---------- ts_collection: type convertible to :py:class:`TimeSeriesCollection` Time series to be written. components: list(str), optional Optional collection of component names to be written. Default is all components. depth: int, optional Depth of component writing. ``depth = 1`` means no components are written. Default is infinity. """ if isinstance(components, list): components = [key.upper() for key in components] temp_ts_collection = convert_to_ts_collection(ts_collection) counter = 0 full_ts_collection = TimeSeriesCollection() while counter < depth: ts_collection = temp_ts_collection names_list = ts_collection.ts_names() if not names_list: break full_ts_collection.add_list(ts_collection) counter += 1 temp_ts_collection = TimeSeriesCollection() for result in names_list: ts = ts_collection.get(result) if isinstance(ts, TimeSeries) and counter < depth: instantiate_components(ts, components, temp_ts_collection) documents = ts_to_dict(full_ts_collection) if documents: bulkop = self.db.initialize_ordered_bulk_op() for document in documents: for key in document: # Some fine tuning in encoding. e.g. MongoDB does not accept numpy.int64. try: if np.issubdtype(document[key], np.signedinteger): document[key] = document[key].item() except: pass try: # Remove values from the document, we are only updating the attributes del document[TS_VALUES] except: pass bulkop.find({ TS_NAME: document[TS_NAME] }).upsert().update({SET: document}) try: return bulkop.execute() except BulkWriteError as bwe: print(bwe.details) write_errors = bwe.details['writeErrors'] input(write_errors) raise
def generate_source_map(ts, external_interfaces): """ Map reading sources with time series. Parameters ---------- ts: :py:class:`TimeSeries` A time series. external_interfaces: list(objects) Instances of external reading classes. Returns ------- dict Dictionary with ``{source_index: time_series_collection}`` pairs. """ source_map = dict() ts_collection = convert_to_ts_collection(ts) for ts in ts_collection: for i, external_interface in enumerate(external_interfaces): if external_interface.is_member(ts): try: source_map[i].add(ts) except KeyError: source_map[i] = TimeSeriesCollection([ts]) break return source_map
def read(self, ts_collection, components=True, depth=np.inf): """ Read time series attributes and values from the database. Parameters ---------- ts_collection: type convertible to :py:class:`TimeSeriesCollection` Time series to be read. components: list(str), optional Optional collection of component names to be read and instantiated. Default is all components. depth: int, optional Depth of component instantiation. ``depth = 1`` means no components are instantiated. Default is infinity. """ if isinstance(components, list): components = [key.upper() for key in components] temp_ts_collection = convert_to_ts_collection(ts_collection) counter = 0 all_names_list = list() while counter < depth: ts_collection = temp_ts_collection names_list = ts_collection.ts_names() if not names_list: break query_bulk_result = list(self.db.find({TS_NAME: {IN: names_list}})) all_names_list += names_list temp_ts_collection = TimeSeriesCollection() counter += 1 new_values_dict = dict() for result in query_bulk_result: ts = ts_collection.get(result[TS_NAME]) if isinstance(ts, TimeSeries): new_values_dict[TS_VALUES] = result[TS_VALUES] del result[TS_VALUES] del result[ID] del result[TS_NAME] ts.update_attributes(result) new_values = pd.Series(new_values_dict[TS_VALUES]) new_values.index = pd.to_datetime(new_values.index, unit='ms', errors='coerce') ts.update_values(new_values) if counter < depth: instantiate_components(ts, components, temp_ts_collection) # Now updating LAST_USE attribute for the requested TimeSeries self.db.update_many({TS_NAME: { IN: all_names_list }}, {SET: { LAST_USE: datetime.datetime.utcnow() }})
def remove(self, ts_collection, components=False, depth=np.inf, confirm=True): """ Remove time series from the database. Parameters ---------- ts_collection: type convertible to :py:class:`TimeSeriesCollection` Time series to be removed. components: list(str), optional Optional collection of component names to be removed. Default is no components. depth: int, optional Depth of component removal. ``depth = 1`` means no components are removed. Default is 1. confirm: bool, optional Confirm before removing. Default is True. """ if isinstance(components, list): components = [key.upper() for key in components] ts_collection = convert_to_ts_collection(ts_collection) counter = 0 full_ts_collection = TimeSeriesCollection() while counter < depth: names_list = ts_collection.ts_names() if not names_list: break full_ts_collection.add_list(ts_collection) counter += 1 ts_collection = TimeSeriesCollection() for result in names_list: ts = full_ts_collection.get(result) if isinstance(ts, TimeSeries): instantiate_components(ts, components, ts_collection) names_to_delete = full_ts_collection.ts_names() if confirm: while True: ans = input( 'Are you sure you want to delete {0} TimeSeries from the {1} database? \n ' '[(y) (n) (l)]'.format(len(names_to_delete), self.collection_name)) ans = ans.lower() if ans == 'l': print(names_to_delete) elif ans == 'n': print("Aborting 'remove' operation.") return elif ans == 'y': break if names_to_delete: return list(self.db.remove({TS_NAME: {IN: names_to_delete}}))
def select(self, **kwargs): """ Get time series names matching attribute specifications. Parameters ---------- kwargs: dict, optional Dictionary of ``{attribute_name: attribute_value}`` pairs to match. Note ---- * Use ``MODE={"OR", "AND"}`` to choose whether to match attributes with "AND" or "OR". Default is "AND". * Use ``AVAILABLE_DATES=list(date-like)`` to return only names of time series that have values at a set of dates. * Use ``ALL_FIELDS=True`` to choose whether to return time series with the "FIELD" attribute. The default behavior is to not return these time series. Returns ------- list(str) Names of time series matching the passed specifications. """ # Loads entire DB, all timeseries (only with names and attributes) and selects by attributes given in arguments. # EXCLUSIVE = 'NO' means it will keep timeseries that do not have one of the attributes. MODE= 'AND' # means it will keep only the timeseries with attributes matching all those given. kwargs = { str(key).upper(): to_list(value) for key, value in kwargs.items() } mode = kwargs.pop("MODE", "AND") all_fields = kwargs.pop("ALL_FIELDS", False) # available_dates = kwargs.pop("AVAILABLE_DATES", None) available_dates = kwargs.pop("AVAILABLE", None) if FIELD not in kwargs and not all_fields: kwargs[FIELD] = [None] if mode.upper() in OR_VALUES: filtered_collection = self.db.find( {OR: [{ i: { IN: list(v) } } for i, v in kwargs.items()]}, {TS_NAME: 1}) elif mode.upper() in AND_VALUES: filtered_collection = self.db.find( {AND: [{ i: { IN: list(v) } } for i, v in kwargs.items()]}, {TS_NAME: 1}) else: filtered_collection = self.db.find({}, {TS_NAME: 1}) new_collection = TimeSeriesCollection() for doc in filtered_collection: new_collection.add(doc[TS_NAME]) if available_dates is not None: available_date = list(map(pd.to_datetime, available_dates)) self.read_values(new_collection) new_collection = TimeSeriesCollection([ ts for ts in new_collection if bool(set(available_date) & set(ts.ts_values.index)) ]) return new_collection
def convert_to_ts_collection(ts): """ Convert a time series list-like object into :py:class:`TimeSeriesCollection` Parameters ---------- ts: :py:class:`TimeSeries`, :py:class:`TimeSeriesCollection`, str, list (str, :py:class:`TimeSeries`) Returns ------- :py:class:`TimeSeriesCollection` Object built with the passed time series list-like. """ if isinstance(ts, TimeSeries): result = TimeSeriesCollection() result.add(ts) return result elif isinstance(ts, str): result = TimeSeriesCollection() result.add(ts) return result elif isinstance(ts, TimeSeriesCollection): return ts elif isinstance(ts, list): result = TimeSeriesCollection() for i in ts: result.add(i) return result
def write(self, ts_collection, components=True, depth=np.inf): """ Write time series attributes and values to the database. Parameters ---------- ts_collection: type convertible to :py:class:`TimeSeriesCollection` Time series to be written. components: list(str), optional Optional collection of component names to be written. Default is all components. depth: int, optional Depth of component writing. ``depth = 1`` means no components are written. Default is infinity. """ if isinstance(components, list): components = [key.upper() for key in components] temp_ts_collection = convert_to_ts_collection(ts_collection) to_updating_collection_names = temp_ts_collection.ts_names()[:] if len(to_updating_collection_names) >= 1000: new_ts_collection = TimeSeriesCollection(temp_ts_collection) temp_ts_collection.remove_list(to_updating_collection_names[998:]) new_ts_collection.remove_list(to_updating_collection_names[:998]) self.write(new_ts_collection, components, depth) counter = 0 full_ts_collection = TimeSeriesCollection() while counter < depth: ts_collection = temp_ts_collection names_list = ts_collection.ts_names() if not names_list: break full_ts_collection.add_list(ts_collection) counter += 1 temp_ts_collection = TimeSeriesCollection() for result in names_list: ts = ts_collection.get(result) if isinstance(ts, TimeSeries) and counter < depth: instantiate_components(ts, components, temp_ts_collection) documents = ts_to_dict(full_ts_collection) if documents: bulkop = self.db.initialize_ordered_bulk_op() for document in documents: for key in document: # Some fine tuning in encoding. e.g. MongoDB does not accept numpy.int64. try: if np.issubdtype(document[key], np.signedinteger): document[key] = document[key].item() except: pass if pd.isnull(document[key]): document[key] = None bulkop.find({ TS_NAME: document[TS_NAME] }).upsert().update({SET: document}) try: retval = bulkop.execute() return retval except BulkWriteError as bwe: warnings.warn(bwe.details) raise bwe
def read_attributes(self, ts_collection, components=True, depth=np.inf, attributes=None): """ Read time series attributes from the database. Parameters ---------- ts_collection: type convertible to :py:class:`TimeSeriesCollection` Time series to be read. components: list(str), optional Optional collection of component names to be read and instantiated. Default is all components. depth: int, optional Depth of component instantiation. ``depth = 1`` means no components are instantiated. Default is infinity. attributes: list(str), optional Optional collection of attribute names to be read. Default is all attributes. """ if isinstance(components, list): components = [key.upper() for key in components] if attributes and attributes is not True: attr_specs = dict() attr_specs[TS_NAME] = 1 for attr in attributes: try: attr = attr.upper() except: pass attr_specs[attr] = 1 else: attr_specs = {TS_VALUES: 0} temp_ts_collection = convert_to_ts_collection(ts_collection) counter = 0 all_names_list = list() while counter < depth: ts_collection = temp_ts_collection names_list = ts_collection.ts_names() if not names_list: break query_bulk_result = list( self.db.find({TS_NAME: { IN: names_list }}, attr_specs)) all_names_list += names_list temp_ts_collection = TimeSeriesCollection() counter += 1 for result in query_bulk_result: ts = ts_collection.get(result[TS_NAME]) if isinstance(ts, TimeSeries): del result[TS_NAME] del result[ID] ts.update_attributes(result) if counter < depth: instantiate_components(ts, components, temp_ts_collection) # Now updating LAST_USE attribute for the requested TimeSeries self.db.update_many({TS_NAME: { IN: all_names_list }}, {SET: { LAST_USE: datetime.datetime.utcnow() }})