def __getFile(self, url, stream_flag=False):
""" returns page as a stream """
cache = CacheManager()
is_cached = cache.checkCache(url)
if is_cached:
return is_cached
response = requests.get(url, stream=stream_flag)
if response.status_code == 200:
cache.saveCache(url, response.content)
return response
else:
raise Exception(url + ' status code: ' + str(response.status_code))
class FileManagerTestCase(BaseTestCase):
def setUp(self):
super().setUp()
self.cache_manager = CacheManager()
self.cache_content = 'this is test content'
self.cache_path = os.path.realpath('data_scraper/cache')
self.md5 = 'b0da4eeb66a12a9ba07cb2f86a967129'
if not os.path.isdir(self.cache_path):
os.mkdir(self.cache_path)
self.file_path = '{}/{}.txt'.format(self.cache_path, self.md5)
file = open(self.file_path, 'w')
file.write(self.cache_content)
file.close()
def tearDown(self):
if os.path.isdir(self.cache_path):
shutil.rmtree(self.cache_path)
super().tearDown()
def test_check_cache(self):
# should return None if cache doesn't exist
self.assertEquals(None,
self.cache_manager.checkCache('http://test.com'))
# should return cache_content if cache does exist
self.assertEquals(
self.cache_content,
self.cache_manager.checkCache('http://httpbin.org'),
)
# should return None if cache file is over 5 days ago
time = datetime.datetime(2015, 1, 1)
time = time.timestamp()
os.utime(self.file_path, (time, time))
self.assertEquals(None,
self.cache_manager.checkCache('http://test.com'))
def test_cache_save(self):
url = 'http://httpbin.org'
self.cache_manager.saveCache(url, self.cache_content)
self.assertTrue(os.path.isfile(self.file_path))
with open(self.file_path, 'r') as file:
self.assertEquals(self.cache_content, file.read())
db_entry = DataCache.query.filter_by(url=url).first()
self.assertIsInstance(db_entry, DataCache)
def setUp(self):
super().setUp()
self.cache_manager = CacheManager()
self.cache_content = 'this is test content'
self.cache_path = os.path.realpath('data_scraper/cache')
self.md5 = 'b0da4eeb66a12a9ba07cb2f86a967129'
if not os.path.isdir(self.cache_path):
os.mkdir(self.cache_path)
self.file_path = '{}/{}.txt'.format(self.cache_path, self.md5)
file = open(self.file_path, 'w')
file.write(self.cache_content)
file.close()
