def test_http_storage_get_dir(tmpdir): with requests_mock.Mocker() as m: m.register_uri("GET", "http://launcher/model/listfile/model0", json=[{ "path": "checkpoint/model.bin", "size": 42 }, { "path": "config.json", "size": 10 }]) m.register_uri( "GET", "http://launcher/model/getfile/model0/checkpoint/model.bin", content=b"model") m.register_uri("GET", "http://launcher/model/getfile/model0/config.json", content=b"config") http = storage.HTTPStorage( "0", "http://launcher/model/getfile/%s", pattern_list="http://launcher/model/listfile/%s") local_dir = tmpdir.join("model0") http.get("model0", str(local_dir), directory=True) assert local_dir.check() assert local_dir.join("checkpoint").join("model.bin").read() == "model" assert local_dir.join("config.json").read() == "config"
def test_http_stream(tmpdir): http = storage.HTTPStorage("0", "http://www.ovh.net/files/%s") size = 0 nchunk = 0 for chunk in http.stream("1Mio.dat"): size += len(chunk) nchunk += 1 assert size == 1024 * 1024 and nchunk == 1024