def generate_time_series(type="const",
parameters={},
start=datetime(2018, 1, 1),
end="now",
frequency=0.1):
timestamps = np.arange(timestamp_to_ms(start), timestamp_to_ms(end),
1000 / frequency).astype(int)
if type == "sine":
return generate_sine(timestamps, parameters)
elif type == "const":
return generate_const(timestamps, parameters)
elif type == "switch": # 0-1
return generate_switch(timestamps, parameters)
else:
raise ValueError("unknown time series type")
def test_retrieve_include_outside_points_paginate_outside_exists(
self, test_time_series, post_spy):
ts = test_time_series[0]
start = datetime(2019, 6, 29)
end = datetime(2019, 6, 29, 5)
test_lim = 2500
dps_non_outside = COGNITE_CLIENT.datapoints.retrieve(id=ts.id,
start=start,
end=end)
tmp = COGNITE_CLIENT.datapoints._DPS_LIMIT
COGNITE_CLIENT.datapoints._DPS_LIMIT = test_lim
dps = COGNITE_CLIENT.datapoints.retrieve(id=ts.id,
start=start,
end=end,
include_outside_points=True)
COGNITE_CLIENT.datapoints._DPS_LIMIT = tmp
ts_outside = set(dps.timestamp) - set(dps_non_outside.timestamp)
assert 2 == len(ts_outside)
for ts in ts_outside:
assert ts < timestamp_to_ms(start) or ts >= timestamp_to_ms(end)
assert set(dps.timestamp) - ts_outside == set(
dps_non_outside.timestamp)
def replicate_datapoints(
client_src: CogniteClient,
client_dst: CogniteClient,
ts_external_id: str,
limit: Optional[int] = None,
partition_size: int = 100000,
mock_run: bool = False,
job_id: int = 1,
src_datapoint_transform: Optional[Callable[[Datapoint], Datapoint]] = None,
timerange_transform: Optional[Callable[[Tuple[int, int]],
Tuple[int, int]]] = None,
start: Union[int, str] = None,
end: Union[int, str] = None,
value_manipulation_lambda_fnc: str = None,
) -> Tuple[bool, int]:
"""
Copies data points from the source tenant into the destination project, for the given time series.
If data points already exist in the destination for the time series, only the newer data points in the source are
copied over.
Args:
client_src: The client corresponding to the source project.
client_dst: The client corresponding to the destination project.
ts_external_id: The external id of the time series to replicate datapoints for
limit: The maximum number of data points to copy
partition_size: The maximum number of datapoints to retrieve per request
mock_run: If true, only retrieves data points from source and does not insert into destination
job_id: The batch number being processed
src_datapoint_transform: Function to apply to all source datapoints before inserting into destination
timerange_transform: Function to set the time range boundaries (start, end) arbitrarily.
start: Timestamp to start replication onwards from; if not specified starts at most recent datapoint
end: If specified, limits replication to datapoints earlier than the end time
value_manipulation_lambda_fnc: A basic lambda function can be provided to manipulate datapoints as a string.
Returns:
A tuple of the success status (True if no failures) and the number of datapoints successfully replicated
"""
try:
latest_dst_dp = client_dst.datapoints.retrieve_latest(
external_id=ts_external_id)
latest_src_dp = client_src.datapoints.retrieve_latest(
external_id=ts_external_id)
except CogniteAPIError as exc:
logging.error(
f"Job {job_id}: Failed for external id {ts_external_id}. {exc}")
return False, 0
if not latest_src_dp:
return True, 0
if src_datapoint_transform:
latest_src_dp = Datapoints(
timestamp=[src_datapoint_transform(latest_src_dp[0]).timestamp])
_start, _end = _get_time_range(latest_src_dp, latest_dst_dp)
start = _start if start is None else timestamp_to_ms(start)
end = _end if end is None else timestamp_to_ms(end)
if timerange_transform:
start, end = timerange_transform(start, end)
# Api Restrictions
start = max(start, 31536000000) # 1971
logging.debug(
f"Job {job_id}: Ext_id: {ts_external_id} Retrieving datapoints between {start} and {end}"
)
datapoints_count = 0
while start < end:
num_to_fetch = partition_size if limit is None else min(
partition_size, limit - datapoints_count)
if num_to_fetch == 0:
break
try:
datapoints = client_src.datapoints.retrieve(
external_id=ts_external_id,
start=start,
end=end,
limit=num_to_fetch)
if not datapoints:
break
if src_datapoint_transform:
transformed_values = []
transformed_timestamps = []
for src_datapoint in datapoints:
transformed_datapoint = src_datapoint_transform(
src_datapoint)
transformed_timestamps.append(
transformed_datapoint.timestamp)
transformed_values.append(transformed_datapoint.value)
datapoints = Datapoints(timestamp=transformed_timestamps,
value=transformed_values)
if value_manipulation_lambda_fnc:
transformed_values = []
transformed_timestamps = []
lambda_fnc = evaluate_lambda_function(
value_manipulation_lambda_fnc)
if lambda_fnc:
for src_datapoint in datapoints:
try:
transformed_timestamps.append(
src_datapoint.timestamp)
transformed_values.append(
lambda_fnc(src_datapoint.value))
except Exception as e:
logging.error(
f"Could not manipulate the datapoint (value={src_datapoint.value},"
+
f" timestamp={src_datapoint.timestamp}). Error: {e}"
)
datapoints = Datapoints(timestamp=transformed_timestamps,
value=transformed_values)
if not mock_run:
client_dst.datapoints.insert(datapoints,
external_id=ts_external_id)
except CogniteAPIError as exc:
logging.error(
f"Job {job_id}: Failed for external id {ts_external_id}. {exc}"
)
return False, datapoints_count
else:
datapoints_count += len(datapoints)
start = datapoints[-1].timestamp + 1
logging.debug(
f"Job {job_id}: Ext_id: {ts_external_id} Number of datapoints: {datapoints_count}"
)
return True, datapoints_count