def load(self, urlpath, output=None, **kwargs):
"""
Downloads data from a given url, generates a hashed filename,
logs metadata, and caches it locally.
Parameters
----------
urlpath: str, location of data
May be a local path, or remote path if including a protocol specifier
such as ``'s3://'``. May include glob wildcards.
output: bool
Whether to show progress bars; turn off for testing
Returns
-------
List of local cache_paths to be opened instead of the remote file(s). If
caching is disable, the urlpath is returned.
"""
if conf.get('cache_disabled', False):
return [urlpath]
self.output = output if output is not None else conf.get(
'cache_download_progress', True)
cache_paths = self._from_metadata(urlpath)
if cache_paths is None:
files_in, files_out = self._make_files(urlpath)
self._load(files_in, files_out, urlpath)
cache_paths = self._from_metadata(urlpath)
return cache_paths
def __init__(self, catalog):
self._catalog = catalog
self._cache = SourceCache()
self._periodic_callbacks = []
auth = conf.get('auth', 'intake.auth.base.BaseAuth')
logger.debug('auth: %s' % auth)
self._auth = remake_instance(auth)
def __init__(self, catalog):
self._catalog = catalog
self._cache = SourceCache()
self._periodic_callbacks = []
auth = conf.get('auth', 'intake.auth.base.BaseAuth')
logger.debug('auth: %s' % auth)
self._auth = get_auth_class(auth['class'], *auth.get('args', tuple()),
**auth.get('kwargs', {}))
def getIntakeURI(cls) -> str:
catalog_paths = iconf.get( "catalog_path" )
if catalog_paths:
catalog_path = catalog_paths[0]
else:
ilDataDir = os.environ.get('ILDATA')
assert ilDataDir is not None, "Must set the ILDATA environment variable to define the data directory"
catalog_path = os.path.join( ilDataDir, "collections", "intake" )
os.makedirs( catalog_path, exist_ok=True )
return catalog_path
def load_combo_catalog():
"""Load a union of the user and global catalogs for convenience"""
user_dir = user_data_dir()
global_dir = global_data_dir()
cat_dirs = []
if os.path.isdir(user_dir):
cat_dirs.append(user_dir + '/*.yaml')
cat_dirs.append(user_dir + '/*.yml')
if os.path.isdir(global_dir):
cat_dirs.append(global_dir + '/*.yaml')
cat_dirs.append(global_dir + '/*.yml')
for path_dir in conf.get('catalog_path', []):
if os.path.isdir(path_dir) and path_dir != '':
cat_dirs.append(path_dir + '/*.yaml')
cat_dirs.append(path_dir + '/*.yml')
return YAMLFilesCatalog(cat_dirs, name='builtin')
def load_combo_catalog():
"""Load a union of the user and global catalogs for convenience"""
user_dir = user_data_dir()
global_dir = global_data_dir()
desc = 'Generated from data packages found on your intake search path'
cat_dirs = []
if os.path.isdir(user_dir):
cat_dirs.append(user_dir + '/*.yaml')
cat_dirs.append(user_dir + '/*.yml')
if os.path.isdir(global_dir):
cat_dirs.append(global_dir + '/*.yaml')
cat_dirs.append(global_dir + '/*.yml')
for path_dir in conf.get('catalog_path', []):
if path_dir != '':
if not path_dir.endswith(('yaml', 'yml')):
cat_dirs.append(path_dir + '/*.yaml')
cat_dirs.append(path_dir + '/*.yml')
else:
cat_dirs.append(path_dir)
return YAMLFilesCatalog(cat_dirs, name='builtin', description=desc)
def main(argv=None):
from intake.config import conf
from intake import open_catalog
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(description='Intake Catalog Server')
parser.add_argument('-p',
'--port',
type=int,
default=conf['port'],
help='port number for server to listen on')
parser.add_argument('--list-entries',
action='store_true',
help='list catalog entries at startup')
parser.add_argument(
'--sys-exit-on-sigterm',
action='store_true',
help='internal flag used during unit testing to ensure '
'.coverage file is written')
parser.add_argument('catalog_args',
metavar='FILE',
type=str,
nargs='+',
help='Name of catalog YAML file')
parser.add_argument('--flatten', dest='flatten', action='store_true')
parser.add_argument('--no-flatten', dest='flatten', action='store_false')
parser.add_argument(
'-a',
'--address',
type=str,
default=conf.get('address', 'localhost'),
help='address to use as a host, defaults to the address '
'in the configuration file, if provided otherwise localhost')
parser.set_defaults(flatten=True)
args = parser.parse_args(argv[1:])
if args.sys_exit_on_sigterm:
signal.signal(signal.SIGTERM, call_exit_on_sigterm)
logger.info('Creating catalog from:')
for arg in args.catalog_args:
logger.info(' - %s' % arg)
catargs = args.catalog_args
if len(catargs) == 1:
catalog = open_catalog(catargs[0])
logger.info("catalog_args: %s" % catargs[0])
else:
catalog = open_catalog(catargs, flatten=args.flatten)
logger.info("catalog_args: %s" % catargs)
if args.list_entries:
# This is not a good idea if the Catalog is huge.
logger.info('Entries:' + ','.join(list(catalog)))
logger.info('Listening on %s:%d' % (args.address, args.port))
server = IntakeServer(catalog)
app = server.make_app()
server.start_periodic_functions(close_idle_after=3600.0)
app.listen(args.port, address=args.address)
try:
tornado.ioloop.IOLoop.current().start()
except KeyboardInterrupt:
logger.critical("Exiting")
except Exception as e:
logger.critical("Exiting due to %s" % e)
