示例#1
0
    def cache_data_file(self, freshen=False):
        """
        Ensures that if a resource comes from a URL that it has been retrieved in its entirety.

        :param freshen: True if this file should be downloaded and the cache obliterated.  False otherwise.

        :return: True if the file has changed since the last time this was called.  False if the file has not changed
                and None if there is no local file to be had (true for PostGIS driver and some custom drivers)
        """

        # immediately return if local caching isn't supported
        if self.resource.resource_file and not freshen:
            _log.debug('using cached file at {0} for {1}'.format(self.resource.resource_file.name, self.resource.slug))
            # if we've cached this file before, it's saved in the resource object.  Ensure that it's on local storage.
            self._ensure_local()
            return False
        else:  # clear the cache and recreate it if we specified "freshen" or if a cached file doesn't exist
            _log.info('processing file to add to cache for {0}'.format(self.resource.slug))
            cache.delete_data_cache(self.resource)
            cache.data_cache_path(self.resource)

            if self.resource.resource_url:
                tempfile = self._fetch_data()
                processed_file, log = self.process_data(tempfile)
            else:
                processed_file, log = self.process_data(self.resource.original_file)

            self.resource.resource_file = processed_file
            self.resource.resource_file.save(self.resource.slug + '.' + self.get_resource_extension(), processed_file)
            self.resource.import_log = log
            self.resource.save()
            self._ensure_local()
            return True
示例#2
0
 def __init__(self, data_resource, **kwargs):
     _log.debug('created driver for {0}'.format(data_resource.slug))
     self.resource = data_resource
     self.cache_path = cache.data_cache_path(data_resource)
     self.cached_basename = os.path.join(self.cache_path, os.path.split(self.resource.slug)[-1])
     self.src_ext = '.sqlite'  # default
     self.ready_data_resource(**kwargs)
示例#3
0
 def __init__(self, data_resource, **kwargs):
     _log.debug('created driver for {0}'.format(data_resource.slug))
     self.resource = data_resource
     self.cache_path = cache.data_cache_path(data_resource)
     self.cached_basename = os.path.join(
         self.cache_path,
         os.path.split(self.resource.slug)[-1])
     self.src_ext = '.sqlite'  # default
     self.ready_data_resource(**kwargs)
示例#4
0
    def cache_data_file(self, freshen=False):
        """
        Ensures that if a resource comes from a URL that it has been retrieved in its entirety.

        :param freshen: True if this file should be downloaded and the cache obliterated.  False otherwise.

        :return: True if the file has changed since the last time this was called.  False if the file has not changed
                and None if there is no local file to be had (true for PostGIS driver and some custom drivers)
        """

        # immediately return if local caching isn't supported
        if self.resource.resource_file and not freshen:
            _log.debug('using cached file at {0} for {1}'.format(
                self.resource.resource_file.name, self.resource.slug))
            # if we've cached this file before, it's saved in the resource object.  Ensure that it's on local storage.
            self._ensure_local()
            return False
        else:  # clear the cache and recreate it if we specified "freshen" or if a cached file doesn't exist
            _log.info('processing file to add to cache for {0}'.format(
                self.resource.slug))
            cache.delete_data_cache(self.resource)
            cache.data_cache_path(self.resource)

            if self.resource.resource_url:
                tempfile = self._fetch_data()
                processed_file, log = self.process_data(tempfile)
            else:
                processed_file, log = self.process_data(
                    self.resource.original_file)

            self.resource.resource_file = processed_file
            self.resource.resource_file.save(
                self.resource.slug + '.' + self.get_resource_extension(),
                processed_file)
            self.resource.import_log = log
            self.resource.save()
            self._ensure_local()
            return True