def load(self, urlpath, output=None, **kwargs): """ Downloads data from a given url, generates a hashed filename, logs metadata, and caches it locally. Parameters ---------- urlpath: str, location of data May be a local path, or remote path if including a protocol specifier such as ``'s3://'``. May include glob wildcards. output: bool Whether to show progress bars; turn off for testing Returns ------- List of local cache_paths to be opened instead of the remote file(s). If caching is disable, the urlpath is returned. """ if conf.get('cache_disabled', False): return [urlpath] self.output = output if output is not None else conf.get( 'cache_download_progress', True) cache_paths = self._from_metadata(urlpath) if cache_paths is None: files_in, files_out = self._make_files(urlpath) self._load(files_in, files_out, urlpath) cache_paths = self._from_metadata(urlpath) return cache_paths
def __init__(self, catalog): self._catalog = catalog self._cache = SourceCache() self._periodic_callbacks = [] auth = conf.get('auth', 'intake.auth.base.BaseAuth') logger.debug('auth: %s' % auth) self._auth = remake_instance(auth)
def __init__(self, catalog): self._catalog = catalog self._cache = SourceCache() self._periodic_callbacks = [] auth = conf.get('auth', 'intake.auth.base.BaseAuth') logger.debug('auth: %s' % auth) self._auth = get_auth_class(auth['class'], *auth.get('args', tuple()), **auth.get('kwargs', {}))
def getIntakeURI(cls) -> str: catalog_paths = iconf.get( "catalog_path" ) if catalog_paths: catalog_path = catalog_paths[0] else: ilDataDir = os.environ.get('ILDATA') assert ilDataDir is not None, "Must set the ILDATA environment variable to define the data directory" catalog_path = os.path.join( ilDataDir, "collections", "intake" ) os.makedirs( catalog_path, exist_ok=True ) return catalog_path
def load_combo_catalog(): """Load a union of the user and global catalogs for convenience""" user_dir = user_data_dir() global_dir = global_data_dir() cat_dirs = [] if os.path.isdir(user_dir): cat_dirs.append(user_dir + '/*.yaml') cat_dirs.append(user_dir + '/*.yml') if os.path.isdir(global_dir): cat_dirs.append(global_dir + '/*.yaml') cat_dirs.append(global_dir + '/*.yml') for path_dir in conf.get('catalog_path', []): if os.path.isdir(path_dir) and path_dir != '': cat_dirs.append(path_dir + '/*.yaml') cat_dirs.append(path_dir + '/*.yml') return YAMLFilesCatalog(cat_dirs, name='builtin')
def load_combo_catalog(): """Load a union of the user and global catalogs for convenience""" user_dir = user_data_dir() global_dir = global_data_dir() desc = 'Generated from data packages found on your intake search path' cat_dirs = [] if os.path.isdir(user_dir): cat_dirs.append(user_dir + '/*.yaml') cat_dirs.append(user_dir + '/*.yml') if os.path.isdir(global_dir): cat_dirs.append(global_dir + '/*.yaml') cat_dirs.append(global_dir + '/*.yml') for path_dir in conf.get('catalog_path', []): if path_dir != '': if not path_dir.endswith(('yaml', 'yml')): cat_dirs.append(path_dir + '/*.yaml') cat_dirs.append(path_dir + '/*.yml') else: cat_dirs.append(path_dir) return YAMLFilesCatalog(cat_dirs, name='builtin', description=desc)
def main(argv=None): from intake.config import conf from intake import open_catalog if argv is None: argv = sys.argv parser = argparse.ArgumentParser(description='Intake Catalog Server') parser.add_argument('-p', '--port', type=int, default=conf['port'], help='port number for server to listen on') parser.add_argument('--list-entries', action='store_true', help='list catalog entries at startup') parser.add_argument( '--sys-exit-on-sigterm', action='store_true', help='internal flag used during unit testing to ensure ' '.coverage file is written') parser.add_argument('catalog_args', metavar='FILE', type=str, nargs='+', help='Name of catalog YAML file') parser.add_argument('--flatten', dest='flatten', action='store_true') parser.add_argument('--no-flatten', dest='flatten', action='store_false') parser.add_argument( '-a', '--address', type=str, default=conf.get('address', 'localhost'), help='address to use as a host, defaults to the address ' 'in the configuration file, if provided otherwise localhost') parser.set_defaults(flatten=True) args = parser.parse_args(argv[1:]) if args.sys_exit_on_sigterm: signal.signal(signal.SIGTERM, call_exit_on_sigterm) logger.info('Creating catalog from:') for arg in args.catalog_args: logger.info(' - %s' % arg) catargs = args.catalog_args if len(catargs) == 1: catalog = open_catalog(catargs[0]) logger.info("catalog_args: %s" % catargs[0]) else: catalog = open_catalog(catargs, flatten=args.flatten) logger.info("catalog_args: %s" % catargs) if args.list_entries: # This is not a good idea if the Catalog is huge. logger.info('Entries:' + ','.join(list(catalog))) logger.info('Listening on %s:%d' % (args.address, args.port)) server = IntakeServer(catalog) app = server.make_app() server.start_periodic_functions(close_idle_after=3600.0) app.listen(args.port, address=args.address) try: tornado.ioloop.IOLoop.current().start() except KeyboardInterrupt: logger.critical("Exiting") except Exception as e: logger.critical("Exiting due to %s" % e)