def test_fetch_resource_succ(self, mock):
mock.get.return_value.content = "test"
mock.get.return_value.status_code = 200
resource = crawler.fetch_resource(Resource(url="http://foo"))
self.assertEquals("test", resource.document)
self.assertEquals(None, resource.last_parsed)
self.assertEquals(None, resource.last_parse_error)
def test_fetch_resource_succ(self, mock):
mock.get.return_value.content = "test"
mock.get.return_value.status_code = 200
resource = crawler.fetch_resource(Resource(url="http://foo"))
self.assertEquals("test", resource.document)
self.assertEquals(None, resource.last_parsed)
self.assertEquals(None, resource.last_parse_error)
def test_fetch_resource_perm_fail(self, mock):
mock.get.return_value.status_code = 404
resource = crawler.fetch_resource(Resource(
url="http://foo",
document=u"stillhere"
))
self.assertEquals(404, resource.last_status_code)
self.assertEquals(u"stillhere", resource.document)
def test_fetch_resource_succ(self, iatikit_mock):
iatikit_mock.return_value = registry
dataset = fac.DatasetFactory.create(
name='old-org-acts',
resources=[fac.ResourceFactory.create(url="http://foo", )])
resource = crawler.fetch_resource(dataset=dataset, ignore_hashes=False)
self.assertEquals(b"<?xml", resource.document[:5])
self.assertEquals(None, resource.last_parsed)
self.assertEquals(None, resource.last_parse_error)
def test_ignore_unchanged_resource(self, iatikit_mock):
iatikit_mock.return_value = registry
with open(
join(dirname(__file__), 'fixtures', 'registry', 'data',
'old-org', 'old-org-acts.xml'), 'rb') as f:
document = f.read()
dataset = fac.DatasetFactory.create(
name='old-org-acts',
resources=[
fac.ResourceFactory.create(
url="https://old-org.nl/sites/default/files/" +
"IATI/activities.xml",
last_parsed=datetime.datetime(2000, 1, 1),
document=document,
)
])
resource = crawler.fetch_resource(dataset=dataset, ignore_hashes=False)
self.assertNotEquals(None, resource.last_parsed)
def test_fetch_resource_perm_fail(self, mock):
mock.get.return_value.status_code = 404
resource = crawler.fetch_resource(
Resource(url="http://foo", document=u"stillhere"))
self.assertEquals(404, resource.last_status_code)
self.assertEquals(u"stillhere", resource.document)