def liquidate(self):
""" Retrieve all files from the cache. Returns a generator"""
if self.is_s3:
for filepath in self.statement():
yield self.s3.download(filepath, self.format)
else:
for filepath in self.statement():
yield load_local(filepath, self.format)
def liquidate(self):
""" Retrieve all files from the cache. Returns a generator"""
if self.is_s3:
for filepath in self.statement():
yield self.s3.download(filepath, self.format)
else:
for filepath in self.statement():
yield load_local(filepath, self.format)
def get_cache(self, url):
""" Wrap requests.get() """
# create a filepath
interval_string = self._gen_interval_string()
filepath = self._url_to_filepath(url, interval_string)
# get cached content
if self.is_s3:
content = self.s3.download(filepath, self.format)
else:
content = load_local(filepath, self.format)
# if it doesen't exist, fetch the url and cache it.
if content is None:
response = requests.get(url)
status_code = response.status_code
if response.status_code != 200:
content = None
else:
# fetch
if self.format == "json":
content = response.json()
elif self.format == "txt":
content = response.content
# cache
if self.is_s3:
self.s3.upload(filepath, content, self.format)
else:
store_local(filepath, content, self.format)
else:
status_code = None
return ATM_Response(
content=content,
url=url,
filepath=filepath,
cache_dir=self.cache_dir,
bucket_name=self.bucket_name,
is_s3=self.is_s3,
status_code=status_code,
source="cache" if status_code is None else "url",
timestamp=int(interval_string) if interval_string else None,
)
def get_cache(self, url):
""" Wrap requests.get() """
# create a filepath
interval_string = self._gen_interval_string()
filepath = self._url_to_filepath(url, interval_string)
# get cached content
if self.is_s3:
content = self.s3.download(filepath, self.format)
else:
content = load_local(filepath, self.format)
# if it doesen't exist, fetch the url and cache it.
if content is None:
response = requests.get(url)
status_code = response.status_code
if response.status_code != 200:
content = None
else:
# fetch
if self.format == "json":
content = response.json()
elif self.format == "txt":
content = response.content
# cache
if self.is_s3:
self.s3.upload(filepath, content, self.format)
else:
store_local(filepath, content, self.format)
else:
status_code = None
return ATM_Response(
content=content,
url=url,
filepath=filepath,
cache_dir=self.cache_dir,
bucket_name=self.bucket_name,
is_s3=self.is_s3,
status_code=status_code,
source="cache" if status_code is None else "url",
timestamp=int(interval_string) if interval_string else None)
def withdraw(self, filepath):
""" Retrieves a file from the cache"""
if self.is_s3:
return self.s3.download(filepath, self.format)
else:
return load_local(filepath, self.format)
def withdraw(self, filepath):
""" Retrieves a file from the cache"""
if self.is_s3:
return self.s3.download(filepath, self.format)
else:
return load_local(filepath, self.format)
