def test_returns_cached_metadata(self):
"""Test if the _get_repo_data helper returns cached data when available."""
repo_data = RepositoryMetadata(
repo_name="python-discord/site",
description="testrepo",
forks=42,
stargazers=42,
language="English",
)
repo_data.save()
metadata = self.home_view._get_repo_data()
self.assertIsInstance(metadata[0], RepositoryMetadata)
self.assertIsInstance(str(metadata[0]), str)
def test_refresh_stale_metadata(self, _: mock.MagicMock):
"""Test if the _get_repo_data helper will refresh when the data is stale."""
repo_data = RepositoryMetadata(
repo_name="python-discord/site",
description="testrepo",
forks=42,
stargazers=42,
language="English",
last_updated=timezone.now() -
timedelta(seconds=HomeView.repository_cache_ttl + 1),
)
repo_data.save()
metadata = self.home_view._get_repo_data()
self.assertIsInstance(metadata[0], RepositoryMetadata)
def test_dont_clean_up_unstale_metadata(self):
"""Tests that we don't clean up good metadata when we start the HomeView."""
repo_data = RepositoryMetadata(
repo_name="python-discord/site",
description="testrepo",
forks=42,
stargazers=42,
language="English",
last_updated=timezone.now() -
timedelta(seconds=HomeView.repository_cache_ttl + 1),
)
repo_data.save()
self.home_view.__init__()
cached_repos = RepositoryMetadata.objects.all()
cached_names = [repo.repo_name for repo in cached_repos]
self.assertIn("python-discord/site", cached_names)
def _get_repo_data(self) -> List[RepositoryMetadata]:
"""Build a list of RepositoryMetadata objects that we can use to populate the front page."""
# First off, load the timestamp of the least recently updated entry.
if self._static_build:
last_update = None
else:
last_update = (RepositoryMetadata.objects.values_list(
"last_updated", flat=True).order_by("last_updated").first())
# If we did not retrieve any results here, we should import them!
if last_update is None:
# Try to get new data from the API. If it fails, we'll return an empty list.
# In this case, we simply don't display our projects on the site.
api_repositories = self._get_api_data()
# Create all the repodata records in the database.
data = [
RepositoryMetadata(
repo_name=api_data["full_name"],
description=api_data["description"],
forks=api_data["forks_count"],
stargazers=api_data["stargazers_count"],
language=api_data["language"],
) for api_data in api_repositories.values()
]
if settings.env("STATIC_BUILD"):
return data
else:
return RepositoryMetadata.objects.bulk_create(data)
# If the data is stale, we should refresh it.
if (timezone.now() - last_update).seconds > self.repository_cache_ttl:
# Try to get new data from the API. If it fails, return the cached data.
api_repositories = self._get_api_data()
if not api_repositories:
return RepositoryMetadata.objects.all()
# Update or create all RepoData objects in self.repos
database_repositories = []
for api_data in api_repositories.values():
repo_data, _created = RepositoryMetadata.objects.update_or_create(
repo_name=api_data["full_name"],
defaults={
'repo_name': api_data["full_name"],
'description': api_data["description"],
'forks': api_data["forks_count"],
'stargazers': api_data["stargazers_count"],
'language': api_data["language"],
})
database_repositories.append(repo_data)
return database_repositories
# Otherwise, if the data is fresher than 2 minutes old, we should just return it.
else:
return RepositoryMetadata.objects.all()
def test_falls_back_to_database_on_error(self, mock_get: mock.MagicMock):
"""Tests that fallback to the database is performed when we get garbage back."""
repo_data = RepositoryMetadata(
repo_name="python-discord/site",
description="testrepo",
forks=42,
stargazers=42,
language="English",
last_updated=timezone.now() -
timedelta(seconds=HomeView.repository_cache_ttl + 1),
)
repo_data.save()
mock_get.return_value.json.return_value = ['garbage']
metadata = self.home_view._get_repo_data()
[item] = metadata
self.assertEqual(item, repo_data)
def _get_repo_data(self) -> List[RepositoryMetadata]:
"""Build a list of RepositoryMetadata objects that we can use to populate the front page."""
# Try to get site data from the cache
try:
repo_data = RepositoryMetadata.objects.get(
repo_name="python-discord/site")
# If the data is stale, we should refresh it.
if (timezone.now() - repo_data.last_updated
).seconds > self.repository_cache_ttl:
# Try to get new data from the API. If it fails, return the cached data.
try:
api_repositories = self._get_api_data()
except (TypeError, ConnectionError):
return RepositoryMetadata.objects.all()
database_repositories = []
# Update or create all RepoData objects in self.repos
for repo_name, api_data in api_repositories.items():
try:
repo_data = RepositoryMetadata.objects.get(
repo_name=repo_name)
repo_data.description = api_data["description"]
repo_data.language = api_data["language"]
repo_data.forks = api_data["forks_count"]
repo_data.stargazers = api_data["stargazers_count"]
except RepositoryMetadata.DoesNotExist:
repo_data = RepositoryMetadata(
repo_name=api_data["full_name"],
description=api_data["description"],
forks=api_data["forks_count"],
stargazers=api_data["stargazers_count"],
language=api_data["language"],
)
repo_data.save()
database_repositories.append(repo_data)
return database_repositories
# Otherwise, if the data is fresher than 2 minutes old, we should just return it.
else:
return RepositoryMetadata.objects.all()
# If this is raised, the database has no repodata at all, we will create them all.
except RepositoryMetadata.DoesNotExist:
database_repositories = []
try:
# Get new data from API
api_repositories = self._get_api_data()
# Create all the repodata records in the database.
for api_data in api_repositories.values():
repo_data = RepositoryMetadata(
repo_name=api_data["full_name"],
description=api_data["description"],
forks=api_data["forks_count"],
stargazers=api_data["stargazers_count"],
language=api_data["language"],
)
repo_data.save()
database_repositories.append(repo_data)
except TypeError:
for repo_name in self.repos:
repo_data = RepositoryMetadata(
last_updated=timezone.now() -
datetime.timedelta(minutes=50),
repo_name=repo_name,
description="Not available.",
forks=999,
stargazers=999,
language="Python",
)
repo_data.save()
database_repositories.append(repo_data)
return database_repositories
def _get_repo_data(self) -> List[RepositoryMetadata]:
"""Build a list of RepositoryMetadata objects that we can use to populate the front page."""
database_repositories = []
# First, let's see if we have any metadata cached.
cached_data = RepositoryMetadata.objects.all()
# If we don't, we have to create some!
if not cached_data:
# Try to get new data from the API. If it fails, we'll return an empty list.
# In this case, we simply don't display our projects on the site.
api_repositories = self._get_api_data()
# Create all the repodata records in the database.
for api_data in api_repositories.values():
repo_data = RepositoryMetadata(
repo_name=api_data["full_name"],
description=api_data["description"],
forks=api_data["forks_count"],
stargazers=api_data["stargazers_count"],
language=api_data["language"],
)
repo_data.save()
database_repositories.append(repo_data)
return database_repositories
# If the data is stale, we should refresh it.
if (timezone.now() - cached_data[0].last_updated
).seconds > self.repository_cache_ttl:
# Try to get new data from the API. If it fails, return the cached data.
api_repositories = self._get_api_data()
if not api_repositories:
return RepositoryMetadata.objects.all()
# Update or create all RepoData objects in self.repos
for repo_name, api_data in api_repositories.items():
try:
repo_data = RepositoryMetadata.objects.get(
repo_name=repo_name)
repo_data.description = api_data["description"]
repo_data.language = api_data["language"]
repo_data.forks = api_data["forks_count"]
repo_data.stargazers = api_data["stargazers_count"]
except RepositoryMetadata.DoesNotExist:
repo_data = RepositoryMetadata(
repo_name=api_data["full_name"],
description=api_data["description"],
forks=api_data["forks_count"],
stargazers=api_data["stargazers_count"],
language=api_data["language"],
)
repo_data.save()
database_repositories.append(repo_data)
return database_repositories
# Otherwise, if the data is fresher than 2 minutes old, we should just return it.
else:
return RepositoryMetadata.objects.all()