def testCopyMetrics_NoNewDatapoints(self, mock_build): # Mock response to API call requesting new data points. mocked_list_result = {} mock_dict = mock.MagicMock() mock_dict.__getitem__.side_effect = (mocked_list_result.__getitem__) (mock_build.return_value.projects.return_value.timeSeries.return_value. list.return_value.execute.return_value) = mocked_list_result utc_now = datetime.utcnow() metrics.copy_metrics('srcProject', 'dstProject', utc_now) # Asset the call to get new data points for each metric. # When there is no entry in LastPoint, start time should be 24 hours ago. start_time = helpers.date_object_to_rfc3339(utc_now - timedelta(days=1)) end_time = helpers.date_object_to_rfc3339(utc_now) for metric in constants.SOURCE_TO_CUSTOM_MAP.keys(): mock_build().projects().timeSeries().list.assert_called_once_with( name='projects/srcProject', interval_startTime=start_time, interval_endTime=end_time, filter='metric.type = "%s"' % metric) # Assert no call was made to write data points. mock_build().projects().timeSeries().create.assert_not_called() # Assert no last point was recorded in datastore. self.assertFalse(schema.LastPoints.query().fetch())
def get_time_series(metric): """Gets time series (data points) for metric and project.""" # Start time is 24h at latest, or endTime of latest data point retrieved. start_time = read_last_point(metric) or helpers.date_object_to_rfc3339( utc_now - timedelta(days=1)) end_time = helpers.date_object_to_rfc3339(utc_now) logging.info( 'Getting time series: metric=%s, startTime=%s, endTime=%s', metric, start_time, end_time) request = service.projects().timeSeries().list( name='projects/%s' % src_project, interval_startTime=start_time, interval_endTime=end_time, filter='metric.type = "%s"' % metric) try: response = request.execute() return response['timeSeries'][ 0] if 'timeSeries' in response else None except HttpError as error: helpers.log_http_error('get_time_series', error) return None
def get(self): """Handler for doing metrics fan-in for all projects.""" # Can only be accessed by cron. if self.request.headers.get('X-Appengine-Cron') is None: self.error(403) return metrics.create_custom_metrics(config.PROJECT_ID) date_string = helpers.date_object_to_rfc3339(datetime.now()) for src_project in metrics.get_projects(config.BILLING_ACCOUNT): taskqueue.add( queue_name='copy-metrics', name=filter(str.isalnum, '%s%s' % (src_project, date_string)), url='/CopyMetrics', method='GET', params={ 'src_project': src_project, 'dst_project': config.PROJECT_ID, })
def testDateObjectToRFC3339(self): date = datetime(2017, 12, 27, 12, 0, 30) self.assertEqual(helpers.date_object_to_rfc3339(date), '2017-12-27T12:00:30Z')
def testCopyMetrics_LastPointExists(self, mock_build): source_metric = constants.SOURCE_TO_CUSTOM_MAP.keys()[0] custom_metric = constants.SOURCE_TO_CUSTOM_MAP[source_metric] utc_now = datetime.utcnow() last_datapoint_time = helpers.date_object_to_rfc3339(utc_now - timedelta( minutes=5)) new_datapoint_time = helpers.date_object_to_rfc3339(utc_now) # Create a last point record. schema.LastPoints( project_id='srcProject', metric=source_metric, date=last_datapoint_time, ).put() # Mock response to API call requesting new data points. point = { 'interval': { 'startTime': new_datapoint_time, 'endTime': new_datapoint_time, }, 'value': { 'int64Value': '1.0', }, } mocked_list_result = { 'timeSeries': [{ 'metric': { 'type': source_metric, }, 'points': [point], }], } mock_dict = mock.MagicMock() mock_dict.__getitem__.side_effect = (mocked_list_result.__getitem__) (mock_build.return_value.projects.return_value.timeSeries.return_value. list.return_value.execute.return_value) = mocked_list_result metrics.copy_metrics('srcProject', 'dstProject', utc_now) # Asset the call to get new data points for each metric. # When there is no entry in LastPoint, start time should be 24 hours ago. start_time = helpers.date_object_to_rfc3339(utc_now - timedelta(days=1)) end_time = helpers.date_object_to_rfc3339(utc_now) for metric in constants.SOURCE_TO_CUSTOM_MAP.keys(): mock_build().projects().timeSeries().list.assert_called_once_with( name='projects/srcProject', interval_startTime=last_datapoint_time, interval_endTime=end_time, filter='metric.type = "%s"' % metric) # Assert the call to write the data points. mock_build().projects().timeSeries().create.assert_called_once_with( name='projects/dstProject', body={ 'timeSeries': [{ 'resource': { 'type': 'global', 'labels': {}, }, 'metric': { 'type': custom_metric, 'labels': { 'project_id': 'srcProject', }, }, 'metricKind': constants.CUSTOM_METRICS_MAP[custom_metric]['metricKind'], 'valueType': constants.CUSTOM_METRICS_MAP[custom_metric]['valueType'], 'points': [point], }], }) # Assert last point was recorded in datastore. self.assertTrue( schema.LastPoints.query( schema.LastPoints.project_id == 'srcProject', schema.LastPoints.metric == source_metric, schema.LastPoints.date == new_datapoint_time).fetch())