def __init__(self, name, aggregates=None, granularity=None, start=None, end=None, limit=None):
self.name = name
self.aggregates = ",".join(aggregates) if aggregates is not None else None
self.granularity = granularity
self.start, self.end = _utils.interval_to_ms(start, end)
if not start:
self.start = None
if not end:
self.end = None
self.limit = limit
def test_interval_to_ms(self):
assert isinstance(utils.interval_to_ms(None, None)[0], int)
assert isinstance(utils.interval_to_ms(None, None)[1], int)
assert isinstance(utils.interval_to_ms('1w-ago', '1d-ago')[0], int)
assert isinstance(utils.interval_to_ms('1w-ago', '1d-ago')[1], int)
assert isinstance(utils.interval_to_ms(datetime(2018, 2, 1), datetime(2018, 3, 1))[0], int)
assert isinstance(utils.interval_to_ms(datetime(2018, 2, 1), datetime(2018, 3, 1))[1], int)
def __init__(self,
tag_id,
aggregates=None,
granularity=None,
start=None,
end=None,
limit=None):
self.tagId = tag_id
self.aggregateFunctions = ",".join(
aggregates) if aggregates is not None else None
self.granularity = granularity
self.start, self.end = _utils.interval_to_ms(start, end)
if not start:
self.start = None
if not end:
self.end = None
self.limit = limit
def get_datapoints_frame(time_series,
aggregates,
granularity,
start=None,
end=None,
**kwargs):
"""Returns a pandas dataframe of datapoints for the given timeseries all on the same timestamps.
This method will automate paging for the user and return all data for the given time period.
Args:
time_series (list): The list of timeseries names to retrieve data for. Each timeseries can be either a string
containing the timeseries or a dictionary containing the names of thetimeseries and a
list of specific aggregate functions.
aggregates (list): The list of aggregate functions you wish to apply to the data for which you have not
specified an aggregate function. Valid aggregate functions are: 'average/avg, max, min,
count, sum, interpolation/int, stepinterpolation/step'.
granularity (str): The granularity of the aggregate values. Valid entries are : 'day/d, hour/h, minute/m,
second/s', or a multiple of these indicated by a number as a prefix e.g. '12hour'.
start (Union[str, int, datetime]): Get datapoints after this time. Format is N[timeunit]-ago where timeunit is w,d,h,m,s.
E.g. '2d-ago' will get everything that is up to 2 days old. Can also send time in ms since
epoch or a datetime object which will be converted to ms since epoch UTC.
end (Union[str, int, datetime]): Get datapoints up to this time. Same format as for start.
Keyword Arguments:
api_key (str): Your api-key.
project (str): Project name.
cookies (dict): Cookies.
limit (str): Max number of rows to return. If limit is specified, this method will not automate
paging and will return a maximum of 100,000 rows.
processes (int): Number of download processes to run in parallell. Defaults to number returned by cpu_count().
Returns:
pandas.DataFrame: A pandas dataframe containing the datapoints for the given timeseries. The datapoints for all the
timeseries will all be on the same timestamps.
Note:
The ``timeseries`` parameter can take a list of strings and/or dicts on the following formats::
Using strings:
['<timeseries1>', '<timeseries2>']
Using dicts:
[{'name': '<timeseries1>', 'aggregates': ['<aggfunc1>', '<aggfunc2>']},
{'name': '<timeseries2>', 'aggregates': []}]
Using both:
['<timeseries1>', {'name': '<timeseries2>', 'aggregates': ['<aggfunc1>', '<aggfunc2>']}]
"""
if not isinstance(time_series, list):
raise _utils.InputError("time_series should be a list")
api_key, project = config.get_config_variables(kwargs.get("api_key"),
kwargs.get("project"))
cookies = config.get_cookies(kwargs.get("cookies"))
start, end = _utils.interval_to_ms(start, end)
if kwargs.get("limit"):
return _get_datapoints_frame_user_defined_limit(
time_series,
aggregates,
granularity,
start,
end,
limit=kwargs.get("limit"),
api_key=api_key,
project=project,
cookies=cookies,
)
diff = end - start
num_of_processes = kwargs.get("processes") or os.cpu_count()
granularity_ms = 1
if granularity:
granularity_ms = _utils.granularity_to_ms(granularity)
# Ensure that number of steps is not greater than the number data points that will be returned
steps = min(num_of_processes, max(1, int(diff / granularity_ms)))
# Make step size a multiple of the granularity requested in order to ensure evenly spaced results
step_size = _utils.round_to_nearest(int(diff / steps), base=granularity_ms)
# Create list of where each of the parallelized intervals will begin
step_starts = [start + (i * step_size) for i in range(steps)]
args = [{
"start": start,
"end": start + step_size
} for start in step_starts]
partial_get_dpsf = partial(
_get_datapoints_frame_helper_wrapper,
time_series=time_series,
aggregates=aggregates,
granularity=granularity,
api_key=api_key,
project=project,
cookies=cookies,
)
if steps == 1:
return _get_datapoints_frame_helper(time_series,
aggregates,
granularity,
start,
end,
api_key=api_key,
project=project,
cookies=cookies)
with Pool(steps) as p:
dataframes = p.map(partial_get_dpsf, args)
df = pd.concat(dataframes).drop_duplicates(subset="timestamp").reset_index(
drop=True)
return df
def get_multi_time_series_datapoints(datapoints_queries,
aggregates=None,
granularity=None,
start=None,
end=None,
**kwargs):
"""Returns a list of DatapointsObjects each of which contains a list of datapoints for the given timeseries.
This method will automate paging for the user and return all data for the given time period(s).
Args:
datapoints_queries (list[v05.dto.DatapointsQuery]): The list of DatapointsQuery objects specifying which
timeseries to retrieve data for.
aggregates (list, optional): The list of aggregate functions you wish to apply to the data. Valid aggregate
functions are: 'average/avg, max, min, count, sum, interpolation/int,
stepinterpolation/step'.
granularity (str): The granularity of the aggregate values. Valid entries are : 'day/d, hour/h,
minute/m, second/s', or a multiple of these indicated by a number as a prefix
e.g. '12hour'.
start (Union[str, int, datetime]): Get datapoints after this time. Format is N[timeunit]-ago where timeunit is w,d,h,m,s.
E.g. '2d-ago' will get everything that is up to 2 days old. Can also send time in ms since
epoch or a datetime object which will be converted to ms since epoch UTC.
end (Union[str, int, datetime]): Get datapoints up to this time. Same format as for start.
Keyword Arguments:
api_key (str): Your api-key.
project (str): Project name.
Returns:
list(v05.dto.DatapointsResponse): A list of data objects containing the requested data with several getter methods
with different output formats.
"""
api_key, project = config.get_config_variables(kwargs.get("api_key"),
kwargs.get("project"))
url = config.get_base_url(
api_version=0.5) + "/projects/{}/timeseries/dataquery".format(project)
start, end = _utils.interval_to_ms(start, end)
num_of_dpqs_with_agg = 0
num_of_dpqs_raw = 0
for dpq in datapoints_queries:
if (dpq.aggregates is None
and aggregates is None) or dpq.aggregates == "":
num_of_dpqs_raw += 1
else:
num_of_dpqs_with_agg += 1
items = []
for dpq in datapoints_queries:
if dpq.aggregates is None and aggregates is None:
dpq.limit = int(_constants.LIMIT / num_of_dpqs_raw)
else:
dpq.limit = int(_constants.LIMIT_AGG / num_of_dpqs_with_agg)
items.append(dpq.__dict__)
body = {
"items": items,
"aggregates": ",".join(aggregates) if aggregates is not None else None,
"granularity": granularity,
"start": start,
"end": end,
}
headers = {
"api-key": api_key,
"content-type": "application/json",
"accept": "application/json"
}
datapoints_responses = []
has_incomplete_requests = True
while has_incomplete_requests:
res = _utils.post_request(
url=url, body=body, headers=headers,
cookies=config.get_cookies()).json()["data"]["items"]
datapoints_responses.append(res)
has_incomplete_requests = False
for i, dpr in enumerate(res):
dpq = datapoints_queries[i]
if len(dpr["datapoints"]) == dpq.limit:
has_incomplete_requests = True
latest_timestamp = dpr["datapoints"][-1]["timestamp"]
ts_granularity = granularity if dpq.granularity is None else dpq.granularity
next_start = latest_timestamp + (_utils.granularity_to_ms(
ts_granularity) if ts_granularity else 1)
else:
next_start = end - 1
if datapoints_queries[i].end:
next_start = datapoints_queries[i].end - 1
datapoints_queries[i].start = next_start
results = [{
"data": {
"items": [{
"name": dpq.name,
"datapoints": []
}]
}
} for dpq in datapoints_queries]
for res in datapoints_responses:
for i, ts in enumerate(res):
results[i]["data"]["items"][0]["datapoints"].extend(
ts["datapoints"])
return DatapointsResponseIterator(
[DatapointsResponse(result) for result in results])
def get_datapoints(name,
aggregates=None,
granularity=None,
start=None,
end=None,
**kwargs):
"""Returns a DatapointsObject containing a list of datapoints for the given query.
This method will automate paging for the user and return all data for the given time period.
Args:
name (str): The name of the timeseries to retrieve data for.
aggregates (list): The list of aggregate functions you wish to apply to the data. Valid aggregate functions
are: 'average/avg, max, min, count, sum, interpolation/int, stepinterpolation/step'.
granularity (str): The granularity of the aggregate values. Valid entries are : 'day/d, hour/h, minute/m,
second/s', or a multiple of these indicated by a number as a prefix e.g. '12hour'.
start (Union[str, int, datetime]): Get datapoints after this time. Format is N[timeunit]-ago where timeunit is w,d,h,m,s.
E.g. '2d-ago' will get everything that is up to 2 days old. Can also send time in ms since
epoch or a datetime object which will be converted to ms since epoch UTC.
end (Union[str, int, datetime]): Get datapoints up to this time. Same format as for start.
Keyword Arguments:
protobuf (bool): Download the data using the binary protobuf format. Only applicable when getting raw data.
Defaults to True.
processes (int): Number of download processes to run in parallell. Defaults to number returned by cpu_count().
api_key (str): Your api-key.
project (str): Project name.
limit (str): Max number of datapoints to return. If limit is specified, this method will not automate
paging and will return a maximum of 100,000 dps.
Returns:
v05.dto.DatapointsResponse: A data object containing the requested data with several getter methods with different
output formats.
"""
api_key, project = config.get_config_variables(kwargs.get("api_key"),
kwargs.get("project"))
start, end = _utils.interval_to_ms(start, end)
if kwargs.get("limit"):
return _get_datapoints_user_defined_limit(
name,
aggregates,
granularity,
start,
end,
limit=kwargs.get("limit"),
protobuf=kwargs.get("protobuf"),
api_key=api_key,
project=project,
)
diff = end - start
num_of_processes = kwargs.get("processes", os.cpu_count())
granularity_ms = 1
if granularity:
granularity_ms = _utils.granularity_to_ms(granularity)
# Ensure that number of steps is not greater than the number data points that will be returned
steps = min(num_of_processes, max(1, int(diff / granularity_ms)))
# Make step size a multiple of the granularity requested in order to ensure evenly spaced results
step_size = _utils.round_to_nearest(int(diff / steps), base=granularity_ms)
# Create list of where each of the parallelized intervals will begin
step_starts = [start + (i * step_size) for i in range(steps)]
args = [{
"start": start,
"end": start + step_size
} for start in step_starts]
partial_get_dps = partial(
_get_datapoints_helper_wrapper,
name=name,
aggregates=aggregates,
granularity=granularity,
protobuf=kwargs.get("protobuf", True),
api_key=api_key,
project=project,
)
if steps == 1:
dps = _get_datapoints_helper(
name,
aggregates,
granularity,
start,
end,
protobuf=kwargs.get("protobuf", True),
api_key=api_key,
project=project,
)
return DatapointsResponse(
{"data": {
"items": [{
"name": name,
"datapoints": dps
}]
}})
with Pool(steps) as p:
datapoints = p.map(partial_get_dps, args)
concat_dps = []
[concat_dps.extend(el) for el in datapoints]
return DatapointsResponse(
{"data": {
"items": [{
"name": name,
"datapoints": concat_dps
}]
}})