def flatten(ts, components=True, depth=np.inf):
""" Create a :py:class:`TimeSeriesCollection` with passed time series and their components.
Parameters
----------
ts: type convertible to :py:obj:`TimeSeriesCollection`
Time series collection whose components are to be "flattened".
components: list(str), optional
Components to include in the final time series collection. Default is all components.
depth: int, optional
Depth of components to include in the final time series collection. Default is infinity.
Returns
-------
:py:obj:`TimeSeriesCollection`
Flattened time series collection.
"""
if isinstance(components, list):
components = [key.upper() for key in components]
flattened = TimeSeriesCollection()
temp_ts_collection = convert_to_ts_collection(ts)
counter = 0
while counter < depth:
ts_collection = temp_ts_collection
if not ts_collection:
break
flattened.add_list(ts_collection)
temp_ts_collection = TimeSeriesCollection()
counter += 1
for ts in ts_collection:
instantiate_components(ts, components, temp_ts_collection)
return flattened
def write_attributes(self, ts_collection, components=True, depth=np.inf):
""" Write time series attributes to the database.
Parameters
----------
ts_collection: type convertible to :py:class:`TimeSeriesCollection`
Time series to be written.
components: list(str), optional
Optional collection of component names to be written. Default is all components.
depth: int, optional
Depth of component writing. ``depth = 1`` means no components are written. Default is infinity.
"""
if isinstance(components, list):
components = [key.upper() for key in components]
temp_ts_collection = convert_to_ts_collection(ts_collection)
counter = 0
full_ts_collection = TimeSeriesCollection()
while counter < depth:
ts_collection = temp_ts_collection
names_list = ts_collection.ts_names()
if not names_list:
break
full_ts_collection.add_list(ts_collection)
counter += 1
temp_ts_collection = TimeSeriesCollection()
for result in names_list:
ts = ts_collection.get(result)
if isinstance(ts, TimeSeries) and counter < depth:
instantiate_components(ts, components, temp_ts_collection)
documents = ts_to_dict(full_ts_collection)
if documents:
bulkop = self.db.initialize_ordered_bulk_op()
for document in documents:
for key in document: # Some fine tuning in encoding. e.g. MongoDB does not accept numpy.int64.
try:
if np.issubdtype(document[key], np.signedinteger):
document[key] = document[key].item()
except:
pass
try:
# Remove values from the document, we are only updating the attributes
del document[TS_VALUES]
except:
pass
bulkop.find({
TS_NAME: document[TS_NAME]
}).upsert().update({SET: document})
try:
return bulkop.execute()
except BulkWriteError as bwe:
print(bwe.details)
write_errors = bwe.details['writeErrors']
input(write_errors)
raise
def generate_source_map(ts, external_interfaces):
""" Map reading sources with time series.
Parameters
----------
ts: :py:class:`TimeSeries`
A time series.
external_interfaces: list(objects)
Instances of external reading classes.
Returns
-------
dict
Dictionary with ``{source_index: time_series_collection}`` pairs.
"""
source_map = dict()
ts_collection = convert_to_ts_collection(ts)
for ts in ts_collection:
for i, external_interface in enumerate(external_interfaces):
if external_interface.is_member(ts):
try:
source_map[i].add(ts)
except KeyError:
source_map[i] = TimeSeriesCollection([ts])
break
return source_map
def read(self, ts_collection, components=True, depth=np.inf):
""" Read time series attributes and values from the database.
Parameters
----------
ts_collection: type convertible to :py:class:`TimeSeriesCollection`
Time series to be read.
components: list(str), optional
Optional collection of component names to be read and instantiated. Default is all components.
depth: int, optional
Depth of component instantiation. ``depth = 1`` means no components are instantiated. Default is infinity.
"""
if isinstance(components, list):
components = [key.upper() for key in components]
temp_ts_collection = convert_to_ts_collection(ts_collection)
counter = 0
all_names_list = list()
while counter < depth:
ts_collection = temp_ts_collection
names_list = ts_collection.ts_names()
if not names_list:
break
query_bulk_result = list(self.db.find({TS_NAME: {IN: names_list}}))
all_names_list += names_list
temp_ts_collection = TimeSeriesCollection()
counter += 1
new_values_dict = dict()
for result in query_bulk_result:
ts = ts_collection.get(result[TS_NAME])
if isinstance(ts, TimeSeries):
new_values_dict[TS_VALUES] = result[TS_VALUES]
del result[TS_VALUES]
del result[ID]
del result[TS_NAME]
ts.update_attributes(result)
new_values = pd.Series(new_values_dict[TS_VALUES])
new_values.index = pd.to_datetime(new_values.index,
unit='ms',
errors='coerce')
ts.update_values(new_values)
if counter < depth:
instantiate_components(ts, components,
temp_ts_collection)
# Now updating LAST_USE attribute for the requested TimeSeries
self.db.update_many({TS_NAME: {
IN: all_names_list
}}, {SET: {
LAST_USE: datetime.datetime.utcnow()
}})
def remove(self,
ts_collection,
components=False,
depth=np.inf,
confirm=True):
""" Remove time series from the database.
Parameters
----------
ts_collection: type convertible to :py:class:`TimeSeriesCollection`
Time series to be removed.
components: list(str), optional
Optional collection of component names to be removed. Default is no components.
depth: int, optional
Depth of component removal. ``depth = 1`` means no components are removed. Default is 1.
confirm: bool, optional
Confirm before removing. Default is True.
"""
if isinstance(components, list):
components = [key.upper() for key in components]
ts_collection = convert_to_ts_collection(ts_collection)
counter = 0
full_ts_collection = TimeSeriesCollection()
while counter < depth:
names_list = ts_collection.ts_names()
if not names_list:
break
full_ts_collection.add_list(ts_collection)
counter += 1
ts_collection = TimeSeriesCollection()
for result in names_list:
ts = full_ts_collection.get(result)
if isinstance(ts, TimeSeries):
instantiate_components(ts, components, ts_collection)
names_to_delete = full_ts_collection.ts_names()
if confirm:
while True:
ans = input(
'Are you sure you want to delete {0} TimeSeries from the {1} database? \n '
'[(y) (n) (l)]'.format(len(names_to_delete),
self.collection_name))
ans = ans.lower()
if ans == 'l':
print(names_to_delete)
elif ans == 'n':
print("Aborting 'remove' operation.")
return
elif ans == 'y':
break
if names_to_delete:
return list(self.db.remove({TS_NAME: {IN: names_to_delete}}))
def select(self, **kwargs):
""" Get time series names matching attribute specifications.
Parameters
----------
kwargs: dict, optional
Dictionary of ``{attribute_name: attribute_value}`` pairs to match.
Note
----
* Use ``MODE={"OR", "AND"}`` to choose whether to match attributes with "AND" or "OR". Default is "AND".
* Use ``AVAILABLE_DATES=list(date-like)`` to return only names of time series that have values at a set of
dates.
* Use ``ALL_FIELDS=True`` to choose whether to return time series with the "FIELD" attribute. The default
behavior is to not return these time series.
Returns
-------
list(str)
Names of time series matching the passed specifications.
"""
# Loads entire DB, all timeseries (only with names and attributes) and selects by attributes given in arguments.
# EXCLUSIVE = 'NO' means it will keep timeseries that do not have one of the attributes. MODE= 'AND'
# means it will keep only the timeseries with attributes matching all those given.
kwargs = {
str(key).upper(): to_list(value)
for key, value in kwargs.items()
}
mode = kwargs.pop("MODE", "AND")
all_fields = kwargs.pop("ALL_FIELDS", False)
# available_dates = kwargs.pop("AVAILABLE_DATES", None)
available_dates = kwargs.pop("AVAILABLE", None)
if FIELD not in kwargs and not all_fields:
kwargs[FIELD] = [None]
if mode.upper() in OR_VALUES:
filtered_collection = self.db.find(
{OR: [{
i: {
IN: list(v)
}
} for i, v in kwargs.items()]}, {TS_NAME: 1})
elif mode.upper() in AND_VALUES:
filtered_collection = self.db.find(
{AND: [{
i: {
IN: list(v)
}
} for i, v in kwargs.items()]}, {TS_NAME: 1})
else:
filtered_collection = self.db.find({}, {TS_NAME: 1})
new_collection = TimeSeriesCollection()
for doc in filtered_collection:
new_collection.add(doc[TS_NAME])
if available_dates is not None:
available_date = list(map(pd.to_datetime, available_dates))
self.read_values(new_collection)
new_collection = TimeSeriesCollection([
ts for ts in new_collection
if bool(set(available_date) & set(ts.ts_values.index))
])
return new_collection
def convert_to_ts_collection(ts):
""" Convert a time series list-like object into :py:class:`TimeSeriesCollection`
Parameters
----------
ts: :py:class:`TimeSeries`, :py:class:`TimeSeriesCollection`, str, list (str, :py:class:`TimeSeries`)
Returns
-------
:py:class:`TimeSeriesCollection`
Object built with the passed time series list-like.
"""
if isinstance(ts, TimeSeries):
result = TimeSeriesCollection()
result.add(ts)
return result
elif isinstance(ts, str):
result = TimeSeriesCollection()
result.add(ts)
return result
elif isinstance(ts, TimeSeriesCollection):
return ts
elif isinstance(ts, list):
result = TimeSeriesCollection()
for i in ts:
result.add(i)
return result
def write(self, ts_collection, components=True, depth=np.inf):
""" Write time series attributes and values to the database.
Parameters
----------
ts_collection: type convertible to :py:class:`TimeSeriesCollection`
Time series to be written.
components: list(str), optional
Optional collection of component names to be written. Default is all components.
depth: int, optional
Depth of component writing. ``depth = 1`` means no components are written. Default is infinity.
"""
if isinstance(components, list):
components = [key.upper() for key in components]
temp_ts_collection = convert_to_ts_collection(ts_collection)
to_updating_collection_names = temp_ts_collection.ts_names()[:]
if len(to_updating_collection_names) >= 1000:
new_ts_collection = TimeSeriesCollection(temp_ts_collection)
temp_ts_collection.remove_list(to_updating_collection_names[998:])
new_ts_collection.remove_list(to_updating_collection_names[:998])
self.write(new_ts_collection, components, depth)
counter = 0
full_ts_collection = TimeSeriesCollection()
while counter < depth:
ts_collection = temp_ts_collection
names_list = ts_collection.ts_names()
if not names_list:
break
full_ts_collection.add_list(ts_collection)
counter += 1
temp_ts_collection = TimeSeriesCollection()
for result in names_list:
ts = ts_collection.get(result)
if isinstance(ts, TimeSeries) and counter < depth:
instantiate_components(ts, components, temp_ts_collection)
documents = ts_to_dict(full_ts_collection)
if documents:
bulkop = self.db.initialize_ordered_bulk_op()
for document in documents:
for key in document: # Some fine tuning in encoding. e.g. MongoDB does not accept numpy.int64.
try:
if np.issubdtype(document[key], np.signedinteger):
document[key] = document[key].item()
except:
pass
if pd.isnull(document[key]):
document[key] = None
bulkop.find({
TS_NAME: document[TS_NAME]
}).upsert().update({SET: document})
try:
retval = bulkop.execute()
return retval
except BulkWriteError as bwe:
warnings.warn(bwe.details)
raise bwe
def read_attributes(self,
ts_collection,
components=True,
depth=np.inf,
attributes=None):
""" Read time series attributes from the database.
Parameters
----------
ts_collection: type convertible to :py:class:`TimeSeriesCollection`
Time series to be read.
components: list(str), optional
Optional collection of component names to be read and instantiated. Default is all components.
depth: int, optional
Depth of component instantiation. ``depth = 1`` means no components are instantiated. Default is infinity.
attributes: list(str), optional
Optional collection of attribute names to be read. Default is all attributes.
"""
if isinstance(components, list):
components = [key.upper() for key in components]
if attributes and attributes is not True:
attr_specs = dict()
attr_specs[TS_NAME] = 1
for attr in attributes:
try:
attr = attr.upper()
except:
pass
attr_specs[attr] = 1
else:
attr_specs = {TS_VALUES: 0}
temp_ts_collection = convert_to_ts_collection(ts_collection)
counter = 0
all_names_list = list()
while counter < depth:
ts_collection = temp_ts_collection
names_list = ts_collection.ts_names()
if not names_list:
break
query_bulk_result = list(
self.db.find({TS_NAME: {
IN: names_list
}}, attr_specs))
all_names_list += names_list
temp_ts_collection = TimeSeriesCollection()
counter += 1
for result in query_bulk_result:
ts = ts_collection.get(result[TS_NAME])
if isinstance(ts, TimeSeries):
del result[TS_NAME]
del result[ID]
ts.update_attributes(result)
if counter < depth:
instantiate_components(ts, components,
temp_ts_collection)
# Now updating LAST_USE attribute for the requested TimeSeries
self.db.update_many({TS_NAME: {
IN: all_names_list
}}, {SET: {
LAST_USE: datetime.datetime.utcnow()
}})