def stream_to_file(self, statement, file, format='csv', fs=None, **kwargs):
"""
Execute a statement against this database and stream results to a file.
This method is a wrapper around `DatabaseClient.stream` that enables the
iterative writing of cursor results to a file. This is especially useful
when there are a very large number of results, and loading them all into
memory would require considerable resources. Note that 'csv' is the
default format for this method (rather than `pandas`).
Args:
statement (str): The statement to be executed against the database.
file (str, file-like-object): The filename where the data should be
written, or an open file-like resource.
format (str): The format to be used ('csv' by default). Format
options can be passed via `**kwargs`.
fs (None, FileSystemClient): The filesystem wihin which the
nominated file should be found. If `None`, the local filesystem
will be used.
**kwargs: Additional keyword arguments to pass onto
`DatabaseClient.stream`.
"""
close_later = False
if isinstance(file, str):
file = (fs or LocalFsClient()).open(file, 'w')
close_later = True
try:
file.writelines(self.stream(statement, format=format, **kwargs))
finally:
if close_later:
file.close()
def query_from_file(self, file, fs=None, **kwargs):
"""
Query using a statement stored in a file.
Args:
file (str, file-like-object): The path of the file containing the
query statement to be executed against the database, or an open
file-like resource.
fs (None, FileSystemClient): The filesystem wihin which the
nominated file should be found. If `None`, the local filesystem
will be used.
**kwargs (dict): Extra keyword arguments to pass on to
`DatabaseClient.query`.
Returns:
object: The results of the query formatted as nominated.
"""
close_later = False
if isinstance(file, str):
file = (fs or LocalFsClient()).open(file, 'r')
close_later = True
try:
return self.query(file.read(), **kwargs)
finally:
if close_later:
file.close()
def _init(self, path, fs=None):
"""
path (str): The top-level path of the cache in the filesystem.
fs (FileSystemClient): The filesystem client to use as the datastore of
this cache. If not specified, this will default to the local filesystem
using `LocalFsClient`.
"""
self.fs = fs or LocalFsClient()
self.path = path
# Currently config is not used, but will be in future versions
self._config = None
self.connection_fields += ('fs',)
def _init(self, path, fs=None):
"""
path (str): The top-level path of the cache in the filesystem.
fs (FileSystemClient, str): The filesystem client to use as the
datastore of this cache. If not specified, this will default to the
local filesystem using `LocalFsClient`. If specified as a string,
and connected to a `DuctRegistry`, upon first use an attempt will be
made to look up a `FileSystemClient` instance in the registry by
this name.
"""
self.fs = fs or LocalFsClient()
self.path = path
# Currently config is not used, but will be in future versions
self._config = None
self.connection_fields += ('fs',)