def test_days_ago(self):
"""Test that the days ago works properly with timezones."""
self.assertEqual(0, days_ago(datetime.now() - timedelta(hours=23)))
self.assertEqual(1, days_ago(datetime.now() - timedelta(hours=24)))
self.assertEqual(
1, days_ago(datetime.now(tz=timezone.utc) - timedelta(hours=47)))
self.assertEqual(
2, days_ago(datetime.now(tz=timezone.utc) - timedelta(hours=48)))
def _include_job(self, job: Job) -> bool:
"""Extend to filter unused jobs."""
if not super()._include_job(job):
return False
max_days = int(cast(str, self._parameter("inactive_job_days")))
actual_days = days_ago(self._latest_build_date_time(job))
return actual_days > max_days
async def test_source_up_to_dateness_without_timestamps(self):
"""Test that the job age in days is returned if the test report doesn't contain timestamps."""
response = await self.collect(get_request_json_side_effect=[
dict(suites=[dict(timestamp=None)]),
dict(timestamp="1565284457173")
])
expected_age = days_ago(datetime.fromtimestamp(1565284457173 / 1000.0))
self.assert_measurement(response, value=str(expected_age))
async def test_source_up_to_dateness(self):
"""Test that the test age is returned."""
html = '<html><table class="config"><tr><td class="name">Start of the test</td>' \
'<td id="start_of_the_test">2019.06.22.06.23.00</td></tr></table></html>'
metric = dict(type="source_up_to_dateness", sources=self.sources, addition="max")
response = await self.collect(metric, get_request_text=html)
expected_age = days_ago(datetime(2019, 6, 22, 6, 23, 0))
self.assert_measurement(response, value=str(expected_age))
async def _parse_value(self, responses: SourceResponses) -> Value:
"""Override to parse the dates from the commits."""
commit_responses = responses[1:]
return str(
days_ago(
max([
parse((await response.json())["committed_date"])
for response in commit_responses
])))
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
commit_responses = responses[1:]
value = str(
days_ago(
max([
parse((await response.json())["committed_date"])
for response in commit_responses
])))
return SourceMeasurement(value=value)
async def test_up_to_dateness(self):
"""Test that the source age in days is returned."""
metric = dict(type="source_up_to_dateness",
addition="max",
sources=self.sources)
response = await self.collect(
metric,
get_request_json_return_value=dict(timestamp="1565284457173"))
expected_age = days_ago(datetime.fromtimestamp(1565284457173 / 1000.))
self.assert_measurement(response, value=str(expected_age))
async def _parse_value(self, responses: SourceResponses) -> Value:
"""Override to parse the date and time of the most recent activity from the cards and lists."""
dates = [self._board.get("createdAt"), self._board.get("modifiedAt")]
for lst in self._lists:
dates.extend([lst.get("createdAt"), lst.get("updatedAt")])
dates.extend([
card["dateLastActivity"]
for card in self._cards.get(lst["_id"], [])
])
return str(days_ago(parse(max([date for date in dates if date]))))
async def _parse_value(self, responses: SourceResponses) -> Value:
"""Override to parse the timestamp from either the job or the test report."""
timestamps = [
suite.get("timestamp")
for suite in (await responses[0].json()).get("suites", [])
if suite.get("timestamp")
]
report_datetime = (parse(
max(timestamps)) if timestamps else datetime.fromtimestamp(
float((await responses[1].json())["timestamp"]) / 1000.0))
return str(days_ago(report_datetime))
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
dates = [self._board.get("createdAt"), self._board.get("modifiedAt")]
for lst in self._lists:
dates.extend([lst.get("createdAt"), lst.get("updatedAt")])
dates.extend([
card["dateLastActivity"]
for card in self._cards.get(lst["_id"], [])
])
return SourceMeasurement(
value=str(days_ago(parse(max([date for date in dates if date])))))
async def _unmerged_branches(
self, responses: SourceResponses) -> List[Dict[str, Any]]:
branches = await responses[0].json()
return [
branch for branch in branches
if not branch["default"] and not branch["merged"]
and days_ago(self._commit_datetime(branch)) > int(
cast(str, self._parameter("inactive_days")))
and not match_string_or_regular_expression(
branch["name"], self._parameter("branches_to_ignore"))
]
async def test_age(self):
"""Test that the age of the file is returned."""
self.sources["source_id"]["parameters"]["repository"] = "repo"
self.sources["source_id"]["parameters"]["file_path"] = "README.md"
metric = dict(type="source_up_to_dateness", sources=self.sources, addition="max")
repositories = dict(value=[dict(id="id", name="repo")])
timestamp = "2019-09-03T20:43:00Z"
commits = dict(value=[dict(committer=dict(date=timestamp))])
response = await self.collect(metric, get_request_json_side_effect=[repositories, commits])
expected_age = str(days_ago(parse(timestamp)))
self.assert_measurement(
response, value=expected_age, landing_url=f"{self.url}/_git/repo?path=README.md&version=GBmaster")
async def _unmerged_branches(self, responses: SourceResponses) -> list[dict[str, Any]]:
"""Override to return a list of unmerged and inactive branches."""
branches = []
for response in responses:
branches.extend(await response.json())
return [
branch
for branch in branches
if not branch["default"]
and not branch["merged"]
and days_ago(self._commit_datetime(branch)) > int(cast(str, self._parameter("inactive_days")))
and not match_string_or_regular_expression(branch["name"], self._parameter("branches_to_ignore"))
]
def setUp(self):
"""Extend to set up test data."""
super().setUp()
self.timestamp = "2019-09-03T20:43:00Z"
self.expected_age = str(days_ago(parse(self.timestamp)))
self.build_json = dict(value=[
dict(
path=r"\\folder",
name="pipeline",
_links=dict(web=dict(href=f"{self.url}/build")),
latestCompletedBuild=dict(result="failed",
finishTime=self.timestamp),
)
])
async def test_source_up_to_dateness(self):
"""Test that the source up-to-dateness of all reports can be measured."""
self.sources["source_id"]["parameters"]["reports"] = []
metric = dict(type="source_up_to_dateness",
sources=self.sources,
addition="sum")
response = await self.collect(
metric, get_request_json_return_value=self.reports)
expected_age = days_ago(parse("2020-06-24T07:53:17+00:00"))
self.assert_measurement(response,
value=str(expected_age),
total="100",
api_url=self.api_url,
landing_url=self.url,
entities=[])
async def test_source_up_to_dateness_report(self):
"""Test that the source up-to-dateness of a specific report can be measured."""
self.sources["source_id"]["parameters"]["reports"] = ["r2"]
metric = dict(type="source_up_to_dateness",
sources=self.sources,
addition="sum")
response = await self.collect(
metric, get_request_json_return_value=self.reports)
expected_age = days_ago(datetime.min)
self.assert_measurement(response,
value=str(expected_age),
total="100",
api_url=self.api_url,
landing_url=self.url,
entities=[])
async def _unmerged_branches(
self, responses: SourceResponses) -> list[dict[str, Any]]:
"""Override to get the unmerged branches response.
Branches are considered unmerged if they have a base branch, have commits that are not on the base branch,
have not been committed to for a minimum number of days, and are not to be ignored.
"""
return [
branch for branch in (await responses[0].json())["value"]
if not branch["isBaseVersion"] and int(branch["aheadCount"]) > 0
and days_ago(self._commit_datetime(branch)) > int(
cast(str, self._parameter("inactive_days")))
and not match_string_or_regular_expression(
branch["name"], self._parameter("branches_to_ignore"))
]
def _count_job(self, job: Job) -> bool:
"""Count the job if its most recent build is too old."""
if super()._count_job(job) and (build_datetime := self._build_datetime(job)) > datetime.min:
max_days = int(cast(str, self._parameter("inactive_days")))
return days_ago(build_datetime) > max_days
def _include_issue(self, issue: Dict) -> bool:
return days_ago(self.__last_test_datetime(issue)) > self.__desired_test_execution_frequency(issue)
def card_is_inactive() -> bool:
"""Return whether the card is inactive."""
date_last_activity = parse(card["dateLastActivity"])
return days_ago(date_last_activity) > int(
cast(int, self._parameter("inactive_days")))
async def _parse_value(self, responses: SourceResponses) -> Value:
"""Override to parse the date and time of the most recent scan."""
scan = (await responses[0].json())[0]
return str(days_ago(parse(scan["dateAndTime"]["finishedOn"])))
async def test_source_up_to_dateness(self):
"""Test that the source up-to-dateness is returned."""
response = await self.collect(get_request_json_return_value=dict(
timestamp="1565284457173"))
expected_age = days_ago(datetime.fromtimestamp(1565284457173 / 1000.0))
self.assert_measurement(response, value=str(expected_age))
def _count_job(self, job: Job) -> bool:
"""Return whether the job is unused."""
max_days = int(cast(str, self._parameter("inactive_job_days")))
return super()._count_job(job) and days_ago(parse(job["created_at"])) > max_days
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
timestamps = [suite.get("timestamp") for suite in (await responses[0].json()).get("suites", [])
if suite.get("timestamp")]
report_datetime = parse(max(timestamps)) if timestamps else \
datetime.fromtimestamp(float((await responses[1].json())["timestamp"]) / 1000.)
return SourceMeasurement(value=str(days_ago(report_datetime)))
async def _unmerged_branches(self, responses: SourceResponses) -> List[Dict[str, Any]]:
return [branch for branch in (await responses[0].json())["value"] if not branch["isBaseVersion"] and
int(branch["aheadCount"]) > 0 and
days_ago(self._commit_datetime(branch)) > int(cast(str, self._parameter("inactive_days"))) and
not match_string_or_regular_expression(branch["name"], self._parameter("branches_to_ignore"))]
def _include_issue(self, issue: Dict) -> bool:
"""Override to only include tests/issues that have been tested too long ago."""
return days_ago(self.__last_test_datetime(issue)) > self.__desired_test_execution_frequency(issue)
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
"""Override to get the datetime from the parse data time method that subclasses should implement."""
date_times = await self._parse_source_response_date_times(responses)
return SourceMeasurement(value=str(days_ago(min(date_times))))
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
scan = (await responses[0].json())[0]
return SourceMeasurement(
value=str(days_ago(parse(scan["dateAndTime"]["finishedOn"]))))
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
date_times = await self._parse_source_response_date_times(responses)
return SourceMeasurement(value=str(days_ago(min(date_times))))
def _ignore_job(self, job: Job) -> bool:
if super()._ignore_job(job):
return True
max_days = int(cast(str, self._parameter("inactive_job_days")))
actual_days = days_ago(self._latest_build_date_time(job))
return actual_days <= max_days
