def test_no_diff(self, db_request, monkeypatch, bq_schema):
project = ProjectFactory.create()
release = ReleaseFactory.create(project=project)
release_file = FileFactory.create(
release=release, filename=f"foobar-{release.version}.tar.gz")
query = pretend.stub(result=pretend.call_recorder(
lambda *a, **kw: [{
"md5_digest": release_file.md5_digest
}]))
get_table = pretend.stub(schema=bq_schema)
bigquery = pretend.stub(
get_table=pretend.call_recorder(lambda t: get_table),
query=pretend.call_recorder(lambda q: query),
)
@pretend.call_recorder
def find_service(name=None):
if name == "gcloud.bigquery":
return bigquery
raise LookupError
db_request.find_service = find_service
db_request.registry.settings = {
"warehouse.release_files_table": "example.pypi.distributions"
}
sync_bigquery_release_files(db_request)
assert db_request.find_service.calls == [
pretend.call(name="gcloud.bigquery")
]
assert bigquery.get_table.calls == [
pretend.call("example.pypi.distributions")
]
assert bigquery.query.calls == [
pretend.call(
"SELECT md5_digest "
"FROM example.pypi.distributions "
f"WHERE md5_digest LIKE '{first}{second}%'", )
for first, second in product("0123456789abcdef", repeat=2)
]
def test_sync_rows(self, db_request, monkeypatch, bq_schema):
project = ProjectFactory.create()
description = DescriptionFactory.create()
release = ReleaseFactory.create(project=project,
description=description)
release.platform = "test_platform"
release_file = FileFactory.create(
release=release,
filename=f"foobar-{release.version}.tar.gz",
md5_digest="01ca4238a0b923820dcc509a6f75849b",
)
release_file2 = FileFactory.create(
release=release,
filename=f"fizzbuzz-{release.version}.tar.gz",
md5_digest="01casd342fb952820dcc509a6f75849b",
)
release._classifiers.append(
ClassifierFactory.create(classifier="foo :: bar"))
release._classifiers.append(
ClassifierFactory.create(classifier="foo :: baz"))
release._classifiers.append(
ClassifierFactory.create(classifier="fiz :: buz"))
DependencyFactory.create(release=release, kind=1)
DependencyFactory.create(release=release, kind=1)
DependencyFactory.create(release=release, kind=2)
DependencyFactory.create(release=release, kind=3)
DependencyFactory.create(release=release, kind=4)
load_config = pretend.call_recorder(lambda *a, **kw: None)
monkeypatch.setattr("warehouse.packaging.tasks.LoadJobConfig",
load_config)
query = pretend.stub(result=pretend.call_recorder(
lambda *a, **kw: [{
"md5_digest": release_file2.md5_digest
}]))
get_table = pretend.stub(schema=bq_schema)
get_result = pretend.stub(result=lambda: None)
bigquery = pretend.stub(
get_table=pretend.call_recorder(lambda t: get_table),
load_table_from_json=pretend.call_recorder(
lambda *a, **kw: get_result),
query=pretend.call_recorder(lambda q: query),
)
@pretend.call_recorder
def find_service(name=None):
if name == "gcloud.bigquery":
return bigquery
raise LookupError
db_request.find_service = find_service
db_request.registry.settings = {
"warehouse.release_files_table": "example.pypi.distributions"
}
sync_bigquery_release_files(db_request)
assert db_request.find_service.calls == [
pretend.call(name="gcloud.bigquery")
]
assert bigquery.get_table.calls == [
pretend.call("example.pypi.distributions")
]
assert bigquery.query.calls == [
pretend.call("SELECT md5_digest "
"FROM example.pypi.distributions "
"WHERE md5_digest LIKE '00%'"),
pretend.call("SELECT md5_digest "
"FROM example.pypi.distributions "
"WHERE md5_digest LIKE '01%'"),
]
assert bigquery.load_table_from_json.calls == [
pretend.call(
[
{
"metadata_version": None,
"name": project.name,
"version": release.version,
"summary": release.summary,
"description": description.raw,
"description_content_type": description.content_type
or None,
"author": release.author or None,
"author_email": release.author_email or None,
"maintainer": release.maintainer or None,
"maintainer_email": release.maintainer_email or None,
"license": release.license or None,
"keywords": release.keywords or None,
"classifiers": release.classifiers or [],
"platform": [release.platform] or [],
"home_page": release.home_page or None,
"download_url": release.download_url or None,
"requires_python": release.requires_python or None,
"requires": release.requires or [],
"provides": release.provides or [],
"obsoletes": release.obsoletes or [],
"requires_dist": release.requires_dist or [],
"provides_dist": release.provides_dist or [],
"obsoletes_dist": release.obsoletes_dist or [],
"requires_external": release.requires_external or [],
"project_urls": release.project_urls or [],
"uploaded_via": release_file.uploaded_via,
"upload_time": release_file.upload_time.isoformat(),
"filename": release_file.filename,
"size": release_file.size,
"path": release_file.path,
"python_version": release_file.python_version,
"packagetype": release_file.packagetype,
"comment_text": release_file.comment_text or None,
"has_signature": release_file.has_signature,
"md5_digest": release_file.md5_digest,
"sha256_digest": release_file.sha256_digest,
"blake2_256_digest": release_file.blake2_256_digest,
},
],
"example.pypi.distributions",
job_config=None,
)
]