def test_public_repo(tempdir):
with working_directory(tempdir.path):
files = pull_github(source=dict(repo="stencila/test", subpath="sub"))
assert list(files.keys()) == ["README.md"]
tempdir.compare(["README.md"])
def test_ok(tempdir):
with working_directory(tempdir.path):
files = pull_plos(
{"article": "10.1371/journal.pcbi.1007273"},
"pcbi.1007273.jats.xml",
)
expected = ["pcbi.1007273.jats.xml", "pcbi.1007273.jats.xml.media/"] + [
"pcbi.1007273.jats.xml.media/" + image for image in [
"e001.png",
"e002.png",
"e003.png",
"e004.png",
"e005.png",
"e006.png",
"g001.png",
"g002.png",
"g003.png",
"g004.png",
"g005.png",
"g006.png",
"g007.png",
"g008.png",
"t001.png",
"t002.png",
]
]
assert files["pcbi.1007273.jats.xml"]["mimetype"] == "application/jats+xml"
assert files["pcbi.1007273.jats.xml.media/e001.png"][
"mimetype"] == "image/png"
tempdir.compare(expected)
def test_one(tempdir):
"""
Test that will create new directories if necessary.
"""
with working_directory(tempdir.path):
files = pull_upload(dict(path=__file__), "a/b/some.txt")
assert list(files.keys()) == ["a/b/some.txt"]
tempdir.compare(["a/", "a/b/", "a/b/some.txt"])
def test_single_file(tempdir):
with working_directory(tempdir.path):
pull_github(
source=dict(repo="stencila/test", subpath="sub/README.md"),
path="sub_README.md",
)
assert not os.path.exists(os.path.join(tempdir.path, "README.md"))
assert not os.path.exists(os.path.join(tempdir.path, "sub/README.md"))
assert os.path.exists(os.path.join(tempdir.path, "sub_README.md"))
def test_ok(tempdir):
with working_directory(tempdir.path):
doc_id = "14z9ScjW4gVjPBRw5XfIdA5LxrApUJx3-S7cXgdNvElc"
doc_json = "{}.json".format(doc_id)
files = pull_gdoc(
source=dict(doc_id=doc_id),
path=doc_json,
secrets=dict(access_token=GOOGLE_TOKEN),
)
assert os.path.exists(doc_json)
assert files[doc_json]["mimetype"] == "application/vnd.google-apps.document"
def test_mergedirs(tempdir):
"""
Test that will merge with existing directories.
"""
with working_directory(tempdir.path):
# A file that may have come from another source
ensure_dir("a/b")
with open("a/b/other.txt", "w") as file:
file.write("whateva")
# Pull an upload source into the same directory
files = pull_upload(dict(path=__file__), "a/b/some.txt")
assert list(files.keys()) == ["a/b/some.txt"]
tempdir.compare(["a/", "a/b/", "a/b/other.txt", "a/b/some.txt"])
def test_overwrite(tempdir):
"""
Test that will overwrite any existing files.
"""
with working_directory(tempdir.path):
with open("some.txt", "w") as file:
file.write("wateva")
assert open("some.txt").read() == "wateva"
files = pull_upload(dict(path=__file__), "some.txt")
assert open("some.txt").read().startswith("import pytest")
assert list(files.keys()) == ["some.txt"]
tempdir.compare(["some.txt"])
def test_ok(tempdir):
with working_directory(tempdir.path):
doc_id = "1SzslazJYVi8KYI6sisUmhJujWW5rbzFthVGtvUb3miM"
doc_json = "{}.json".format(doc_id)
files = pull_gsheet(
source=dict(doc_id=doc_id),
path=doc_json,
secrets=dict(access_token=ACCESS_TOKEN),
)
assert os.path.exists(doc_json)
assert files[doc_json][
"mimetype"] == "application/vnd.google-apps.spreadsheet"
def test_extension_from_mimetype(tempdir):
with working_directory(tempdir.path):
files = pull_http({"url": "https://httpbin.org/get"})
assert files["get.json"]["mimetype"] == "application/json"
files = pull_http({"url": "https://httpbin.org/image/png"},
path="image")
assert files["image.png"]["mimetype"] == "image/png"
files = pull_http({"url": "https://httpbin.org/html"}, path="content")
assert files["content.html"]["mimetype"] == "text/html"
files = pull_http({"url": "https://httpbin.org/html"}, path="foo.bar")
assert files["foo.bar"]["mimetype"] is None
def test_to_gdoc(tempdir):
with working_directory(tempdir.path):
with open("input.md", "w") as file:
file.write("Hello world!")
job = Convert()
result = job.do(
input="input.md",
output="output.gdoc",
secrets=dict(access_token=ACCESS_TOKEN),
)
tempdir.compare(["input.md", "output.gdoc"])
assert result["output.gdoc"][
"mimetype"] == "application/vnd.google-apps.document"
def test_large_zip(tempdir):
with working_directory(tempdir.path):
zip_file = os.path.join(os.path.dirname(__file__), "fixtures",
"1000-maniacs.zip")
files = pull_zip(zip_file, strip=0)
assert len(files.keys()) == 1000
info = files.get("1.txt")
del info["modified"]
assert info == {
"encoding": None,
"fingerprint":
"7f8cafc8690f2707d1b7e8a40a59d8299e1af8f862576ba896acaca287fd4006",
"mimetype": "text/plain",
"size": 6,
}
def test_huge_zip(tempdir):
"""
A performance test using a huge repository archive.
To run this test, download the Zip file of https://github.com/sophiedeb/logistic_models
and run this file with:
./venv/bin/pytest jobs/pull/githubtest.py --durations=0
It has 3304 files in it (at time of writing). Before optimizations this too 360s to
run; after 11s.
"""
zip_file = os.path.join(os.path.dirname(__file__), "fixtures",
"logistic_models-master.zip")
if not os.path.exists(zip_file):
return
with working_directory(tempdir.path):
pull_zip(zip_file)
def test_subdirectory(tempdir):
path = "a/very/very/deep/sub/directory"
with working_directory(tempdir.path):
files = pull_github(
source=dict(repo="stencila/test", subpath=""),
path=path,
)
filenames = [
".travis.yml",
"README.md",
"document.md",
"sub/README.md",
]
assert sorted(list(files.keys())) == sorted(filenames)
for expected in filenames:
assert os.path.exists(os.path.join(tempdir.path, path, expected))