def test_retrieve_monthly_downloads():
project = Project(ProjectName("random"), Downloads(10))
with freezegun.freeze_time("2020-04-12"):
project.add_downloads(date(2020, 3, 9), "0.0.1", DayDownloads(20, 20))
project.add_downloads(date(2020, 4, 10), "0.0.1", DayDownloads(10, 10))
project.add_downloads(date(2020, 4, 11), "0.0.1", DayDownloads(15, 15))
assert project.month_downloads() == Downloads(25)
def get(self, project_name: str) -> Optional[Project]:
normalized_name = ProjectName(project_name).name
project_data = self._client.projects.find_one({"name": normalized_name})
if project_data is None:
return None
project = Project(ProjectName(project_data["name"]), Downloads(project_data["total_downloads"]))
if "downloads" in project_data:
downloads = sorted(project_data["downloads"].items(), key=lambda x: x[0])
for iso_date, version_downloads in downloads:
for r in version_downloads:
date = datetime.date.fromisoformat(iso_date)
version = r[0]
project.add_downloads(date, version, DayDownloads(r[1], 0))
project._repository_saved_downloads.add((iso_date, version))
# Don't count the downloads twice
project.total_downloads -= Downloads(r[1])
else:
raw_downloads = self._client.project_downloads.find({"project": normalized_name})
downloads = sorted(raw_downloads, key=lambda x: x["date"])
for day_downloads in downloads:
for version_downloads in day_downloads["downloads"]:
pip_downlods = version_downloads["pip_downloads"] if "pip_downlods" in version_downloads else 0
project.add_downloads(
datetime.date.fromisoformat(day_downloads["date"]),
version_downloads["version"],
DayDownloads(version_downloads["downloads"], pip_downlods),
)
# Don't count the downloads twice
project.total_downloads -= Downloads(version_downloads["downloads"])
return project
def test_project_replace_downloads():
project = Project(ProjectName("random"), Downloads(10))
date = datetime.now().date()
version = "2.3.1"
project.add_downloads(date, version, DayDownloads(25, 25))
project.add_downloads(date, version, DayDownloads(5, 5))
assert project.total_downloads == Downloads(15)
assert project.last_downloads() == [
ProjectVersionDownloads(date, version, Downloads(5), Downloads(5))
]
assert project.versions() == {version}
def test_add_downloads_to_project():
project = Project(ProjectName("random"), Downloads(0))
date = datetime.now().date()
version = "2.3.1"
day_downloads = DayDownloads(10, 10)
project.add_downloads(date, version, day_downloads)
assert project.total_downloads == day_downloads.total_downloads()
assert project.last_downloads() == [
ProjectVersionDownloads(date, version, Downloads(10), Downloads(10))
]
assert project.versions() == {version}
def test_save_many_projects_with_new_format(mongo_client: MongoClient,
repository: ProjectRepository):
project = Project(ProjectName("climoji"), Downloads(100))
project.add_downloads(datetime.date(2020, 3, 31), "2.0",
DayDownloads(40, 10))
project.add_downloads(datetime.date(2020, 3, 31), "2.0.1",
DayDownloads(30, 10))
project.add_downloads(datetime.date(2020, 4, 1), "2.0",
DayDownloads(20, 10))
repository.save_projects([project])
data = mongo_client.pepy_test.projects.find_one(
{"name": project.name.name})
expected_data = {
"name": "climoji",
"total_downloads": 190,
"monthly_downloads": 0
}
for key, value in expected_data.items():
assert key in data
assert value == data[key]
downloads_data = sorted(mongo_client.pepy_test.project_downloads.find(
{"project": project.name.name}),
key=lambda x: x["date"])
expected_downloads_data = [
{
"project":
"climoji",
"date":
"2020-03-31",
"downloads": [{
"version": "2.0",
"downloads": 40
}, {
"version": "2.0.1",
"downloads": 30
}],
},
{
"project": "climoji",
"date": "2020-04-01",
"downloads": [{
"version": "2.0",
"downloads": 20
}]
},
]
assert len(expected_downloads_data) == len(downloads_data)
for i in range(len(expected_downloads_data)):
for key, value in expected_downloads_data[i].items():
assert key in downloads_data[i]
assert value == downloads_data[i][key]
def test_update_min_date_when_no_other_downloads():
project = Project(ProjectName("random"), Downloads(10))
project.add_downloads(date(2019, 3, 9), "0.0.6", DayDownloads(20, 20))
project.add_downloads(date(2020, 4, 10), "0.0.2", DayDownloads(10, 10))
project.add_downloads(date(2020, 4, 10), "0.0.4", DayDownloads(10, 10))
assert project.total_downloads == Downloads(50)
assert project.last_downloads() == [
ProjectVersionDownloads(date(2020, 4, 10), "0.0.2", Downloads(10),
Downloads(10)),
ProjectVersionDownloads(date(2020, 4, 10), "0.0.4", Downloads(10),
Downloads(10)),
]
assert project.versions() == {"0.0.6", "0.0.2", "0.0.4"}
assert project.min_date == date(2020, 4, 10)
def handle(self, cmd: UpdateVersionDownloads):
if not self._admin_password_checker.check(cmd.password):
self._logger.info("Invalid password")
raise InvalidAdminPassword(cmd.password)
self._logger.info(f"Getting downloads from date {cmd.date}...")
stats_result = self._stats_viewer.get_version_downloads(cmd.date)
self._logger.info(
f"Retrieved {stats_result.total_rows} downloads. Saving to db...")
start_time = timeit.default_timer()
batch_iterator = 0
total_batches = int(stats_result.total_rows / 250)
for batch in self._batch(stats_result.rows, 250):
self._logger.info(f"Batch {batch_iterator} of {total_batches}")
batch_iterator += 1
projects = {}
for row in batch:
if row.project in projects:
project = projects.get(row.project)
else:
project = self._project_repository.get(
row.project, downloads_from=cmd.date)
if project is None:
project = Project(ProjectName(row.project), Downloads(0))
project.add_downloads(
row.date, row.version,
DayDownloads(row.downloads, row.pip_downloads))
projects[row.project] = project
self._project_repository.save_projects(list(projects.values()))
end_time = timeit.default_timer()
self._logger.info(
f"Total downloads updated. Total time + {(end_time - start_time):.4f} seconds"
)
def test_remove_old_data():
project = Project(ProjectName("random"), Downloads(10))
old_date = datetime.now().date() - timedelta(days=181)
limit_date = datetime.now().date() - timedelta(days=180)
now_date = datetime.now().date()
project.add_downloads(old_date, "2.3.1", DayDownloads(10, 10))
project.add_downloads(limit_date, "2.3.0", DayDownloads(20, 20))
project.add_downloads(now_date, "2.3.2", DayDownloads(30, 30))
assert project.total_downloads == Downloads(70)
assert project.last_downloads() == [
ProjectVersionDownloads(limit_date, "2.3.0", Downloads(20),
Downloads(20)),
ProjectVersionDownloads(now_date, "2.3.2", Downloads(30),
Downloads(30)),
]
assert {"2.3.0", "2.3.2"}.issubset(project.versions())
def test_do_not_touch_already_saved_data(mongo_client: MongoClient,
repository: ProjectRepository):
# Used for performance reasons
data = {
"name": "climoji",
"total_downloads": 1100,
}
query = {"name": "climoji"}
mongo_client.pepy_test.projects.replace_one(query, data, upsert=True)
downloads_data = [
InsertOne({
"project": "climoji",
"date": "2020-04-01",
"downloads": [{
"version": "2.0",
"downloads": 30
}]
}),
]
mongo_client.pepy_test.project_downloads.bulk_write(downloads_data)
project = repository.get("climoji")
project.add_downloads(datetime.date(2020, 4, 1), "2.0", DayDownloads(1, 1))
repository.save(project)
downloads_data = sorted(mongo_client.pepy_test.project_downloads.find(
{"project": project.name.name}),
key=lambda x: x["date"])
expected_downloads_data = [{
"project":
"climoji",
"date":
"2020-04-01",
"downloads": [{
"version": "2.0",
"downloads": 30
}]
}]
assert len(expected_downloads_data) == len(downloads_data)
for i in range(len(expected_downloads_data)):
for key, value in expected_downloads_data[i].items():
assert key in downloads_data[i]
assert value == downloads_data[i][key]