def test_create_existing_archive(self):
"""Test if create method fails when the given archive path already exists"""
archive_path = os.path.join(self.test_path, 'myarchive')
Archive.create(archive_path)
with self.assertRaisesRegex(ArchiveError, "archive %s already exists" % archive_path):
Archive.create(archive_path)
def test_store(self):
"""Test whether data is properly stored in the archive"""
data_requests = [
("https://example.com/", {'q': 'issues', 'date': '2017-01-10'}, {}),
("https://example.com/", {'q': 'issues', 'date': '2018-01-01'}, {}),
("https://example.com/tasks", {'task_id': 10}, {'Accept': 'application/json'}),
]
httpretty.register_uri(httpretty.GET,
"https://example.com/",
body='{"hey": "there"}',
status=200)
httpretty.register_uri(httpretty.GET,
"https://example.com/tasks",
body='{"task": "my task"}',
status=200)
archive_path = os.path.join(self.test_path, 'myarchive')
archive = Archive.create(archive_path)
# Store data in the archive
responses = []
for dr in data_requests:
response = requests.get(dr[0], params=dr[1], headers=dr[2])
archive.store(dr[0], dr[1], dr[2], response)
responses.append(response)
db = sqlite3.connect(archive.archive_path)
cursor = db.cursor()
cursor.execute("SELECT hashcode, data, uri, payload, headers FROM archive")
data_stored = cursor.fetchall()
cursor.close()
self.assertEqual(len(data_stored), len(data_requests))
ds = data_stored[0]
dr = data_requests[0]
self.assertEqual(ds[0], '0fa4ce047340780f08efca92f22027514263521d')
self.assertEqual(pickle.loads(ds[1]).url, responses[0].url)
self.assertEqual(ds[2], dr[0])
self.assertEqual(pickle.loads(ds[3]), dr[1])
self.assertEqual(pickle.loads(ds[4]), dr[2])
ds = data_stored[1]
dr = data_requests[1]
self.assertEqual(ds[0], '3879a6f12828b7ac3a88b7167333e86168f2f5d2')
self.assertEqual(pickle.loads(ds[1]).url, responses[1].url)
self.assertEqual(ds[2], dr[0])
self.assertEqual(pickle.loads(ds[3]), dr[1])
self.assertEqual(pickle.loads(ds[4]), dr[2])
ds = data_stored[2]
dr = data_requests[2]
self.assertEqual(ds[0], 'ef38f574a0745b63a056e7befdb7a06e7cf1549b')
self.assertEqual(pickle.loads(ds[1]).url, responses[2].url)
self.assertEqual(ds[2], dr[0])
self.assertEqual(pickle.loads(ds[3]), dr[1])
self.assertEqual(pickle.loads(ds[4]), dr[2])
def test_fetch_from_archive_exception(self):
"""Test whether serialized exceptions are thrown"""
archive_path = os.path.join(self.test_path, 'myarchive')
archive = Archive.create(archive_path)
httpretty.register_uri(httpretty.GET,
CLIENT_SPIDERMAN_URL,
body="bad",
status=404)
# populate the archive and check that an exception is thown when fetching data from the API
client = MockedClient(CLIENT_API_URL,
sleep_time=0.1,
max_retries=1,
archive=archive)
with self.assertRaises(requests.exceptions.HTTPError):
_ = client.fetch(CLIENT_SPIDERMAN_URL)
# retrieve data from the archive and check that an exception is
# thown as happened when fetching data from the API)
client = MockedClient(CLIENT_API_URL,
sleep_time=0.1,
max_retries=1,
archive=archive,
from_archive=True)
with self.assertRaises(requests.exceptions.HTTPError):
_ = client.fetch(CLIENT_SPIDERMAN_URL)
def test_fetch_from_archive(self):
"""Test whether responses are correctly fecthed from an archive"""
archive_path = os.path.join(self.test_path, 'myarchive')
archive = Archive.create(archive_path)
httpretty.register_uri(httpretty.GET,
CLIENT_SUPERMAN_URL,
body="good",
status=200)
client = MockedClient(CLIENT_API_URL,
sleep_time=0.1,
max_retries=1,
archive=archive)
answer_api = client.fetch(CLIENT_SUPERMAN_URL)
client = MockedClient(CLIENT_API_URL,
sleep_time=0.1,
max_retries=1,
archive=archive,
from_archive=True)
answer_archive = client.fetch(CLIENT_SUPERMAN_URL)
self.assertEqual(answer_api.text, answer_archive.text)
def test_retrieve_missing(self):
"""Test whether the retrieval of non archived data throws an error
In the exceptional case of a failure in retrieving data from an archive (e.g., manual modification),
an exception is thrown to stop the retrieval from the archive
"""
archive_path = os.path.join(self.test_path, 'myarchive')
archive = Archive.create(archive_path)
with self.assertRaisesRegex(ArchiveError, "not found in archive"):
_ = archive.retrieve("http://wrong", payload={}, headers={})
def test_init_archive(self):
"""Test whether the archive is properly initialized when executing the fetch method"""
archive_path = os.path.join(self.test_path, 'myarchive')
archive = Archive.create(archive_path)
b = MockedBackend('test', archive=archive)
_ = [item for item in b.fetch()]
self.assertEqual(b.archive.backend_name, b.__class__.__name__)
self.assertEqual(b.archive.backend_version, b.version)
self.assertEqual(b.archive.origin, b.origin)
self.assertEqual(b.archive.category, MockedBackend.CATEGORY)
def test_init(self):
"""Test whether an archive is propertly initialized"""
archive_path = os.path.join(self.test_path, 'myarchive')
_ = Archive.create(archive_path)
archive = Archive(archive_path)
self.assertEqual(archive.archive_path, archive_path)
self.assertEqual(archive.created_on, None)
self.assertEqual(archive.origin, None)
self.assertEqual(archive.backend_name, None)
self.assertEqual(archive.backend_version, None)
self.assertEqual(archive.category, None)
self.assertEqual(archive.backend_params, None)
def test_archive(self):
"""Test whether archive value is initializated"""
archive_path = os.path.join(self.test_path, 'myarchive')
archive = Archive.create(archive_path)
b = Backend('test', archive=archive)
self.assertEqual(b.archive, archive)
b = Backend('test')
self.assertEqual(b.archive, None)
b.archive = archive
self.assertEqual(b.archive, archive)
def test_retrieve(self):
"""Test whether data is properly retrieved from the archive"""
url = "https://example.com/tasks"
payload = {'task_id': 10}
headers = {'Accept': 'application/json'}
httpretty.register_uri(httpretty.GET,
url,
body='{"hey": "there"}',
status=200)
response = requests.get(url, params=payload, headers=headers)
archive_path = os.path.join(self.test_path, 'myarchive')
archive = Archive.create(archive_path)
archive.store(url, payload, headers, response)
data = archive.retrieve(url, payload, headers)
self.assertEqual(data.url, response.url)
def test_store_duplicate(self):
"""Test whether the insertion of duplicated data throws an error"""
url = "https://example.com/tasks"
payload = {'task_id': 10}
headers = {'Accept': 'application/json'}
httpretty.register_uri(httpretty.GET,
url,
body='{"hey": "there"}',
status=200)
response = requests.get(url, params=payload, headers=headers)
archive_path = os.path.join(self.test_path, 'myarchive')
archive = Archive.create(archive_path)
archive.store(url, payload, headers, response)
# check the unique index filters duplicated API calls
with self.assertRaisesRegex(ArchiveError, "duplicated entry"):
archive.store(url, payload, headers, response)
def test_init_metadata(self):
"""Test whether metadata information is properly initialized"""
archive_path = os.path.join(self.test_path, 'myarchive')
archive = Archive.create(archive_path)
before_dt = datetime_to_utc(datetime_utcnow())
archive.init_metadata('marvel.com', 'marvel-comics-backend', '0.1.0',
'issue', {'from_date': before_dt})
after_dt = datetime_to_utc(datetime_utcnow())
archive_copy = Archive(archive_path)
# Both copies should have the same parameters
for arch in [archive, archive_copy]:
self.assertEqual(arch.origin, 'marvel.com')
self.assertEqual(arch.backend_name, 'marvel-comics-backend')
self.assertEqual(arch.backend_version, '0.1.0')
self.assertEqual(arch.category, 'issue')
self.assertGreaterEqual(arch.created_on, before_dt)
self.assertLessEqual(arch.created_on, after_dt)
self.assertDictEqual(arch.backend_params, {'from_date': before_dt})
def test_create(self):
"""Test a new an empty archive is created"""
archive_path = os.path.join(self.test_path, 'myarchive')
archive = Archive.create(archive_path)
# Archive file was created
self.assertEqual(archive.archive_path, archive_path)
self.assertEqual(os.path.exists(archive.archive_path), True)
# Properties are initialized
self.assertEqual(archive.created_on, None)
self.assertEqual(archive.origin, None)
self.assertEqual(archive.backend_name, None)
self.assertEqual(archive.backend_version, None)
self.assertEqual(archive.category, None)
self.assertEqual(archive.backend_params, None)
# Tables are empty
nrows = count_number_rows(archive_path, Archive.ARCHIVE_TABLE)
self.assertEqual(nrows, 0)
nrows = count_number_rows(archive_path, Archive.METADATA_TABLE)
self.assertEqual(nrows, 0)
def setUp(self):
self.test_path = tempfile.mkdtemp(prefix='perceval-mozilla_')
archive_path = os.path.join(self.test_path, 'myarchive')
self.archive = Archive.create(archive_path)