def copy_test_file(filename, target_dir):
if not os.path.exists(target_dir):
os.makedirs(target_dir) # can't use exist_ok in Python 2.7
target_dir = make_path_posix(target_dir)
# Put a catalog file in the user catalog directory
test_dir = make_path_posix(os.path.dirname(__file__))
test_catalog = posixpath.join(test_dir, filename)
target_catalog = posixpath.join(target_dir, '__unit_test_' + filename)
shutil.copyfile(test_catalog, target_catalog)
return target_catalog
def __init__(self,
driver,
spec,
catdir=None,
cache_dir=None,
storage_options={}):
"""
Parameters
----------
driver: str
Name of the plugin that can load catalog entry
spec: dict
Specification for caching the data source.
cache_dir: str or None
Explicit location of cache root directory
catdir: str or None
Directory containing the catalog from which this spec was made
"""
self._driver = driver
self._spec = spec
cd = make_path_posix(cache_dir or conf['cache_dir'])
if cd == 'catdir':
if catdir is None:
raise TypeError('cache_dir="catdir" only allowed when loaded'
'from a catalog file.')
cd = posixpath.join(catdir, 'intake_cache')
self._cache_dir = cd
self._storage_options = storage_options
self._metadata = CacheMetadata()
def tempdir():
import tempfile
import shutil
d = make_path_posix(str(tempfile.mkdtemp()))
try:
yield d
finally:
shutil.rmtree(d)
def data_filenames():
basedir = make_path_posix(os.path.dirname(__file__))
return dict(sample1=posixpath.join(basedir, 'sample1.csv'),
sample2_1=posixpath.join(basedir, 'sample2_1.csv'),
sample2_2=posixpath.join(basedir, 'sample2_2.csv'),
sample2_all=posixpath.join(basedir, 'sample2_*.csv'),
sample_pattern=posixpath.join(basedir,
'sample{num:d}_{dup:d}.csv'))
def sample_pattern_datasource_with_cache(data_filenames):
metadata = {
'cache': [{
'argkey': 'urlpath',
'regex': make_path_posix(os.path.dirname(__file__)),
'type': 'file'
}]
}
return csv.CSVSource(data_filenames['sample_pattern'], metadata=metadata)
def multi_server(tmpdir):
fn1 = make_path_posix(os.path.join(tmpdir, 'cat1.yaml'))
shutil.copyfile(catalog_file, fn1)
fn2 = make_path_posix(os.path.join(tmpdir, 'cat2.yaml'))
shutil.copyfile(catalog_file, fn2)
P = subprocess.Popen(['intake-server', fn1, fn2, '--no-flatten'])
t = time.time()
while True:
try:
requests.get('http://localhost:5000')
yield 'intake://localhost:5000'
break
except:
time.sleep(0.2)
if time.time() - t > 10:
break
P.terminate()
P.wait()
shutil.rmtree(tmpdir)
def unbzip(f, outpath):
import bz2
z = bz2.open(f)
fn = os.path.basename(f)[:-3]
with open(os.path.join(outpath, fn), 'wb') as fout:
data = True
while data:
data = z.read(2**15)
fout.write(data)
return [make_path_posix(os.path.join(outpath, fn))]
def unzip(f, outpath):
import zipfile
z = zipfile.ZipFile(f, 'r')
z.extractall(outpath)
out = [
make_path_posix(os.path.join(outpath, fn.filename))
for fn in z.filelist
]
z.close()
return out
def untar(f, outpath):
import tarfile
tar = tarfile.open(f, "r:")
out = [
make_path_posix(os.path.join(outpath, fn.name))
for fn in tar.getmembers()
]
tar.extractall(outpath)
tar.close()
return out
def temp_cache(tempdir):
import intake
old = intake.config.conf.copy()
olddir = intake.config.confdir
intake.config.confdir = tempdir
intake.config.conf.update({'cache_dir': make_path_posix(str(tempdir)),
'cache_download_progress': False,
'cache_disabled': False})
intake.config.save_conf()
try:
yield
finally:
intake.config.confdir = olddir
intake.config.conf.update(old)
def __init__(self, *args, **kwargs):
from intake import config
self._path = posixpath.join(make_path_posix(config.confdir),
'cache_metadata.json')
d = os.path.dirname(self._path)
if not os.path.exists(d):
os.makedirs(d)
if os.path.isfile(self._path):
with open(self._path) as f:
self._metadata = json.load(f)
else:
self._metadata = {}
def sanitize_path(path):
"Utility for cleaning up paths."
storage_option = infer_storage_options(path)
protocol = storage_option['protocol']
if protocol in ('http', 'https'):
# Most FSs remove the protocol but not HTTPFS. We need to strip
# it to match properly.
path = os.path.normpath(path.replace("{}://".format(protocol), ''))
elif protocol == 'file':
# Just removing trailing slashes from file paths.
path = os.path.normpath(path)
# Otherwise we just make sure that path is posix
return make_path_posix(path)
def test_ds_set_cache_dir(catalog_cache):
cat = catalog_cache['test_cache']()
defaults = cat.cache_dirs
new_cache_dir = os.path.join(os.getcwd(), 'test_cache_dir')
cat.set_cache_dir(new_cache_dir)
cache = cat.cache[0]
assert make_path_posix(cache._cache_dir) == make_path_posix(new_cache_dir)
cache_paths = cache.load(cat._urlpath, output=False)
cache_path = cache_paths[-1]
expected_cache_dir = make_path_posix(new_cache_dir)
assert expected_cache_dir in cache_path
assert defaults[0] not in cache_path
assert os.path.isfile(cache_path)
cache_id = os.path.basename(os.path.dirname(cache_path))
# Checking for md5 hash
assert all(c in string.hexdigits for c in cache_id)
cache.clear_all()
shutil.rmtree(expected_cache_dir)
def _make_files(self, urlpath, **kwargs):
import tempfile
d = tempfile.mkdtemp()
from dask.bytes import open_files
self._ensure_cache_dir()
self._urlpath = urlpath
files_in = open_files(urlpath, 'rb')
files_out = [open_files(
[make_path_posix(os.path.join(d, os.path.basename(f.path)))],
'wb', **self._storage_options)[0]
for f in files_in
]
super(CompressedCache, self)._load(files_in, files_out, urlpath,
meta=False)
return files_in, files_out
def global_data_dir():
"""Return the global Intake catalog dir for the current environment"""
prefix = False
if VIRTUALENV_VAR in os.environ:
prefix = os.environ[VIRTUALENV_VAR]
elif CONDA_VAR in os.environ:
prefix = sys.prefix
elif which('conda'):
# conda exists but is not activated
prefix = conda_prefix()
if prefix:
# conda and virtualenv use Linux-style directory pattern
return make_path_posix(os.path.join(prefix, 'share', 'intake'))
else:
return appdirs.site_data_dir(appname='intake', appauthor='intake')
def temp_cache(tempdir):
import intake
from intake.container.persist import store
old = intake.config.conf.copy()
olddir = intake.config.confdir
intake.config.confdir = tempdir
intake.config.conf.update({'cache_dir': make_path_posix(str(tempdir)),
'cache_download_progress': False,
'cache_disabled': False})
intake.config.conf.save()
store.__init__(os.path.join(tempdir, 'persist'))
try:
yield
finally:
intake.config.confdir = olddir
intake.config.conf.update(old)
intake.config.save_conf()
def address_server(tmpdir):
fn1 = make_path_posix(os.path.join(tmpdir, 'cat1.yaml'))
shutil.copyfile(catalog_file, fn1)
P = subprocess.Popen(['intake-server','--port', '5001', '--address', '0.0.0.0', fn1])
t = time.time()
while True:
try:
requests.get('http://0.0.0.0:5001')
yield 'intake://0.0.0.0:5001'
break
except:
time.sleep(0.2)
if time.time() - t > 10:
break
P.terminate()
P.wait()
shutil.rmtree(tmpdir)
def port_server(tmpdir):
fn1 = make_path_posix(os.path.join(tmpdir, 'cat1.yaml'))
shutil.copyfile(catalog_file, fn1)
port = free_port()
P = subprocess.Popen(['intake-server', '--port', str(port), fn1])
t = time.time()
try:
while True:
try:
requests.get('http://localhost:%s' % port)
yield 'intake://localhost:%s' % port
break
except:
time.sleep(0.2)
if time.time() - t > 10:
break
finally:
P.terminate()
P.wait()
shutil.rmtree(tmpdir)
def test_roundtrip_file_path():
path = os.path.dirname(__file__)
actual = make_path_posix(path)
assert '\\' not in actual
assert os.path.samefile(actual, path)
def test_noops(path):
"""For non windows style paths, make_path_posix should be a noop"""
assert make_path_posix(path) == path
def test_make_path_posix_removes_double_sep():
path = 'user//fake.file'
actual = make_path_posix(path)
expected = 'user/fake.file'
assert actual == expected
def test_windows_file_path():
path = 'C:\\Users\\user\\fake.file'
actual = make_path_posix(path)
expected = 'C:/Users/user/fake.file'
assert actual == expected
# Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors
# All rights reserved.
#
# The full license is in the LICENSE file, distributed with this software.
#-----------------------------------------------------------------------------
import intake
import intake.config
from intake.source.cache import CacheMetadata
import os
import pytest
import subprocess
import sys
from intake.utils import make_path_posix
cpath = make_path_posix(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', '..', '..', 'catalog',
'tests', 'catalog_caching.yml')))
@pytest.mark.skipif(sys.version_info[0] == 2,
reason="Py2 exists early on argparse")
def test_help(env):
out = subprocess.check_output(['intake', 'cache'],
env=env,
universal_newlines=True)
assert out.startswith('usage: ')
out2 = subprocess.check_output(['intake', 'cache', '-h'],
env=env,
universal_newlines=True)
assert out2 == out
import requests
import shutil
import subprocess
import time
from tornado.ioloop import IOLoop
from tornado.testing import AsyncHTTPTestCase
import msgpack
from intake import Catalog, open_catalog
from intake.container.serializer import MsgPackSerializer, GzipCompressor
from intake.cli.server.server import IntakeServer
from intake.compat import unpack_kwargs
from intake.utils import make_path_posix
catalog_file = make_path_posix(
os.path.join(os.path.dirname(__file__), 'catalog1.yml'))
class TestServerV1Base(AsyncHTTPTestCase):
def get_app(self):
local_catalog = Catalog(catalog_file)
self.server = IntakeServer(local_catalog)
return self.server.make_app()
def encode(self, msg):
return msgpack.packb(msg, use_bin_type=True)
def decode(self, bytestr):
return msgpack.unpackb(bytestr, **unpack_kwargs)
def abspath(filename):
return make_path_posix(
os.path.join(os.path.dirname(__file__), filename))
