def cache_data_file(self, freshen=False):
"""
Ensures that if a resource comes from a URL that it has been retrieved in its entirety.
:param freshen: True if this file should be downloaded and the cache obliterated. False otherwise.
:return: True if the file has changed since the last time this was called. False if the file has not changed
and None if there is no local file to be had (true for PostGIS driver and some custom drivers)
"""
# immediately return if local caching isn't supported
if self.resource.resource_file and not freshen:
_log.debug('using cached file at {0} for {1}'.format(self.resource.resource_file.name, self.resource.slug))
# if we've cached this file before, it's saved in the resource object. Ensure that it's on local storage.
self._ensure_local()
return False
else: # clear the cache and recreate it if we specified "freshen" or if a cached file doesn't exist
_log.info('processing file to add to cache for {0}'.format(self.resource.slug))
cache.delete_data_cache(self.resource)
cache.data_cache_path(self.resource)
if self.resource.resource_url:
tempfile = self._fetch_data()
processed_file, log = self.process_data(tempfile)
else:
processed_file, log = self.process_data(self.resource.original_file)
self.resource.resource_file = processed_file
self.resource.resource_file.save(self.resource.slug + '.' + self.get_resource_extension(), processed_file)
self.resource.import_log = log
self.resource.save()
self._ensure_local()
return True
def __init__(self, data_resource, **kwargs):
_log.debug('created driver for {0}'.format(data_resource.slug))
self.resource = data_resource
self.cache_path = cache.data_cache_path(data_resource)
self.cached_basename = os.path.join(self.cache_path, os.path.split(self.resource.slug)[-1])
self.src_ext = '.sqlite' # default
self.ready_data_resource(**kwargs)
def __init__(self, data_resource, **kwargs):
_log.debug('created driver for {0}'.format(data_resource.slug))
self.resource = data_resource
self.cache_path = cache.data_cache_path(data_resource)
self.cached_basename = os.path.join(
self.cache_path,
os.path.split(self.resource.slug)[-1])
self.src_ext = '.sqlite' # default
self.ready_data_resource(**kwargs)
def cache_data_file(self, freshen=False):
"""
Ensures that if a resource comes from a URL that it has been retrieved in its entirety.
:param freshen: True if this file should be downloaded and the cache obliterated. False otherwise.
:return: True if the file has changed since the last time this was called. False if the file has not changed
and None if there is no local file to be had (true for PostGIS driver and some custom drivers)
"""
# immediately return if local caching isn't supported
if self.resource.resource_file and not freshen:
_log.debug('using cached file at {0} for {1}'.format(
self.resource.resource_file.name, self.resource.slug))
# if we've cached this file before, it's saved in the resource object. Ensure that it's on local storage.
self._ensure_local()
return False
else: # clear the cache and recreate it if we specified "freshen" or if a cached file doesn't exist
_log.info('processing file to add to cache for {0}'.format(
self.resource.slug))
cache.delete_data_cache(self.resource)
cache.data_cache_path(self.resource)
if self.resource.resource_url:
tempfile = self._fetch_data()
processed_file, log = self.process_data(tempfile)
else:
processed_file, log = self.process_data(
self.resource.original_file)
self.resource.resource_file = processed_file
self.resource.resource_file.save(
self.resource.slug + '.' + self.get_resource_extension(),
processed_file)
self.resource.import_log = log
self.resource.save()
self._ensure_local()
return True