def liquidate(self): """ Retrieve all files from the cache. Returns a generator""" if self.is_s3: for filepath in self.statement(): yield self.s3.download(filepath, self.format) else: for filepath in self.statement(): yield load_local(filepath, self.format)
def get_cache(self, url): """ Wrap requests.get() """ # create a filepath interval_string = self._gen_interval_string() filepath = self._url_to_filepath(url, interval_string) # get cached content if self.is_s3: content = self.s3.download(filepath, self.format) else: content = load_local(filepath, self.format) # if it doesen't exist, fetch the url and cache it. if content is None: response = requests.get(url) status_code = response.status_code if response.status_code != 200: content = None else: # fetch if self.format == "json": content = response.json() elif self.format == "txt": content = response.content # cache if self.is_s3: self.s3.upload(filepath, content, self.format) else: store_local(filepath, content, self.format) else: status_code = None return ATM_Response( content=content, url=url, filepath=filepath, cache_dir=self.cache_dir, bucket_name=self.bucket_name, is_s3=self.is_s3, status_code=status_code, source="cache" if status_code is None else "url", timestamp=int(interval_string) if interval_string else None, )
def get_cache(self, url): """ Wrap requests.get() """ # create a filepath interval_string = self._gen_interval_string() filepath = self._url_to_filepath(url, interval_string) # get cached content if self.is_s3: content = self.s3.download(filepath, self.format) else: content = load_local(filepath, self.format) # if it doesen't exist, fetch the url and cache it. if content is None: response = requests.get(url) status_code = response.status_code if response.status_code != 200: content = None else: # fetch if self.format == "json": content = response.json() elif self.format == "txt": content = response.content # cache if self.is_s3: self.s3.upload(filepath, content, self.format) else: store_local(filepath, content, self.format) else: status_code = None return ATM_Response( content=content, url=url, filepath=filepath, cache_dir=self.cache_dir, bucket_name=self.bucket_name, is_s3=self.is_s3, status_code=status_code, source="cache" if status_code is None else "url", timestamp=int(interval_string) if interval_string else None)
def withdraw(self, filepath): """ Retrieves a file from the cache""" if self.is_s3: return self.s3.download(filepath, self.format) else: return load_local(filepath, self.format)