def copy_test_file(filename, target_dir): if not os.path.exists(target_dir): os.makedirs(target_dir) # can't use exist_ok in Python 2.7 target_dir = make_path_posix(target_dir) # Put a catalog file in the user catalog directory test_dir = make_path_posix(os.path.dirname(__file__)) test_catalog = posixpath.join(test_dir, filename) target_catalog = posixpath.join(target_dir, '__unit_test_' + filename) shutil.copyfile(test_catalog, target_catalog) return target_catalog
def __init__(self, driver, spec, catdir=None, cache_dir=None, storage_options={}): """ Parameters ---------- driver: str Name of the plugin that can load catalog entry spec: dict Specification for caching the data source. cache_dir: str or None Explicit location of cache root directory catdir: str or None Directory containing the catalog from which this spec was made """ self._driver = driver self._spec = spec cd = make_path_posix(cache_dir or conf['cache_dir']) if cd == 'catdir': if catdir is None: raise TypeError('cache_dir="catdir" only allowed when loaded' 'from a catalog file.') cd = posixpath.join(catdir, 'intake_cache') self._cache_dir = cd self._storage_options = storage_options self._metadata = CacheMetadata()
def tempdir(): import tempfile import shutil d = make_path_posix(str(tempfile.mkdtemp())) try: yield d finally: shutil.rmtree(d)
def data_filenames(): basedir = make_path_posix(os.path.dirname(__file__)) return dict(sample1=posixpath.join(basedir, 'sample1.csv'), sample2_1=posixpath.join(basedir, 'sample2_1.csv'), sample2_2=posixpath.join(basedir, 'sample2_2.csv'), sample2_all=posixpath.join(basedir, 'sample2_*.csv'), sample_pattern=posixpath.join(basedir, 'sample{num:d}_{dup:d}.csv'))
def sample_pattern_datasource_with_cache(data_filenames): metadata = { 'cache': [{ 'argkey': 'urlpath', 'regex': make_path_posix(os.path.dirname(__file__)), 'type': 'file' }] } return csv.CSVSource(data_filenames['sample_pattern'], metadata=metadata)
def multi_server(tmpdir): fn1 = make_path_posix(os.path.join(tmpdir, 'cat1.yaml')) shutil.copyfile(catalog_file, fn1) fn2 = make_path_posix(os.path.join(tmpdir, 'cat2.yaml')) shutil.copyfile(catalog_file, fn2) P = subprocess.Popen(['intake-server', fn1, fn2, '--no-flatten']) t = time.time() while True: try: requests.get('http://localhost:5000') yield 'intake://localhost:5000' break except: time.sleep(0.2) if time.time() - t > 10: break P.terminate() P.wait() shutil.rmtree(tmpdir)
def unbzip(f, outpath): import bz2 z = bz2.open(f) fn = os.path.basename(f)[:-3] with open(os.path.join(outpath, fn), 'wb') as fout: data = True while data: data = z.read(2**15) fout.write(data) return [make_path_posix(os.path.join(outpath, fn))]
def unzip(f, outpath): import zipfile z = zipfile.ZipFile(f, 'r') z.extractall(outpath) out = [ make_path_posix(os.path.join(outpath, fn.filename)) for fn in z.filelist ] z.close() return out
def untar(f, outpath): import tarfile tar = tarfile.open(f, "r:") out = [ make_path_posix(os.path.join(outpath, fn.name)) for fn in tar.getmembers() ] tar.extractall(outpath) tar.close() return out
def temp_cache(tempdir): import intake old = intake.config.conf.copy() olddir = intake.config.confdir intake.config.confdir = tempdir intake.config.conf.update({'cache_dir': make_path_posix(str(tempdir)), 'cache_download_progress': False, 'cache_disabled': False}) intake.config.save_conf() try: yield finally: intake.config.confdir = olddir intake.config.conf.update(old)
def __init__(self, *args, **kwargs): from intake import config self._path = posixpath.join(make_path_posix(config.confdir), 'cache_metadata.json') d = os.path.dirname(self._path) if not os.path.exists(d): os.makedirs(d) if os.path.isfile(self._path): with open(self._path) as f: self._metadata = json.load(f) else: self._metadata = {}
def sanitize_path(path): "Utility for cleaning up paths." storage_option = infer_storage_options(path) protocol = storage_option['protocol'] if protocol in ('http', 'https'): # Most FSs remove the protocol but not HTTPFS. We need to strip # it to match properly. path = os.path.normpath(path.replace("{}://".format(protocol), '')) elif protocol == 'file': # Just removing trailing slashes from file paths. path = os.path.normpath(path) # Otherwise we just make sure that path is posix return make_path_posix(path)
def test_ds_set_cache_dir(catalog_cache): cat = catalog_cache['test_cache']() defaults = cat.cache_dirs new_cache_dir = os.path.join(os.getcwd(), 'test_cache_dir') cat.set_cache_dir(new_cache_dir) cache = cat.cache[0] assert make_path_posix(cache._cache_dir) == make_path_posix(new_cache_dir) cache_paths = cache.load(cat._urlpath, output=False) cache_path = cache_paths[-1] expected_cache_dir = make_path_posix(new_cache_dir) assert expected_cache_dir in cache_path assert defaults[0] not in cache_path assert os.path.isfile(cache_path) cache_id = os.path.basename(os.path.dirname(cache_path)) # Checking for md5 hash assert all(c in string.hexdigits for c in cache_id) cache.clear_all() shutil.rmtree(expected_cache_dir)
def _make_files(self, urlpath, **kwargs): import tempfile d = tempfile.mkdtemp() from dask.bytes import open_files self._ensure_cache_dir() self._urlpath = urlpath files_in = open_files(urlpath, 'rb') files_out = [open_files( [make_path_posix(os.path.join(d, os.path.basename(f.path)))], 'wb', **self._storage_options)[0] for f in files_in ] super(CompressedCache, self)._load(files_in, files_out, urlpath, meta=False) return files_in, files_out
def global_data_dir(): """Return the global Intake catalog dir for the current environment""" prefix = False if VIRTUALENV_VAR in os.environ: prefix = os.environ[VIRTUALENV_VAR] elif CONDA_VAR in os.environ: prefix = sys.prefix elif which('conda'): # conda exists but is not activated prefix = conda_prefix() if prefix: # conda and virtualenv use Linux-style directory pattern return make_path_posix(os.path.join(prefix, 'share', 'intake')) else: return appdirs.site_data_dir(appname='intake', appauthor='intake')
def temp_cache(tempdir): import intake from intake.container.persist import store old = intake.config.conf.copy() olddir = intake.config.confdir intake.config.confdir = tempdir intake.config.conf.update({'cache_dir': make_path_posix(str(tempdir)), 'cache_download_progress': False, 'cache_disabled': False}) intake.config.conf.save() store.__init__(os.path.join(tempdir, 'persist')) try: yield finally: intake.config.confdir = olddir intake.config.conf.update(old) intake.config.save_conf()
def address_server(tmpdir): fn1 = make_path_posix(os.path.join(tmpdir, 'cat1.yaml')) shutil.copyfile(catalog_file, fn1) P = subprocess.Popen(['intake-server','--port', '5001', '--address', '0.0.0.0', fn1]) t = time.time() while True: try: requests.get('http://0.0.0.0:5001') yield 'intake://0.0.0.0:5001' break except: time.sleep(0.2) if time.time() - t > 10: break P.terminate() P.wait() shutil.rmtree(tmpdir)
def port_server(tmpdir): fn1 = make_path_posix(os.path.join(tmpdir, 'cat1.yaml')) shutil.copyfile(catalog_file, fn1) port = free_port() P = subprocess.Popen(['intake-server', '--port', str(port), fn1]) t = time.time() try: while True: try: requests.get('http://localhost:%s' % port) yield 'intake://localhost:%s' % port break except: time.sleep(0.2) if time.time() - t > 10: break finally: P.terminate() P.wait() shutil.rmtree(tmpdir)
def test_roundtrip_file_path(): path = os.path.dirname(__file__) actual = make_path_posix(path) assert '\\' not in actual assert os.path.samefile(actual, path)
def test_noops(path): """For non windows style paths, make_path_posix should be a noop""" assert make_path_posix(path) == path
def test_make_path_posix_removes_double_sep(): path = 'user//fake.file' actual = make_path_posix(path) expected = 'user/fake.file' assert actual == expected
def test_windows_file_path(): path = 'C:\\Users\\user\\fake.file' actual = make_path_posix(path) expected = 'C:/Users/user/fake.file' assert actual == expected
# Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. #----------------------------------------------------------------------------- import intake import intake.config from intake.source.cache import CacheMetadata import os import pytest import subprocess import sys from intake.utils import make_path_posix cpath = make_path_posix( os.path.abspath( os.path.join(os.path.dirname(__file__), '..', '..', '..', 'catalog', 'tests', 'catalog_caching.yml'))) @pytest.mark.skipif(sys.version_info[0] == 2, reason="Py2 exists early on argparse") def test_help(env): out = subprocess.check_output(['intake', 'cache'], env=env, universal_newlines=True) assert out.startswith('usage: ') out2 = subprocess.check_output(['intake', 'cache', '-h'], env=env, universal_newlines=True) assert out2 == out
import requests import shutil import subprocess import time from tornado.ioloop import IOLoop from tornado.testing import AsyncHTTPTestCase import msgpack from intake import Catalog, open_catalog from intake.container.serializer import MsgPackSerializer, GzipCompressor from intake.cli.server.server import IntakeServer from intake.compat import unpack_kwargs from intake.utils import make_path_posix catalog_file = make_path_posix( os.path.join(os.path.dirname(__file__), 'catalog1.yml')) class TestServerV1Base(AsyncHTTPTestCase): def get_app(self): local_catalog = Catalog(catalog_file) self.server = IntakeServer(local_catalog) return self.server.make_app() def encode(self, msg): return msgpack.packb(msg, use_bin_type=True) def decode(self, bytestr): return msgpack.unpackb(bytestr, **unpack_kwargs)
def abspath(filename): return make_path_posix( os.path.join(os.path.dirname(__file__), filename))