def test_last_changed_datetime(self):
resource = fac.ResourceFactory.create(
url=u"http://test",
document="""
<iati-activities>
<iati-activity>
<iati-identifier>test_deleted_activity</iati-identifier>
<title>test_deleted_activity</title>
<reporting-org ref="GB-CHC-202918" type="21">Oxfam GB</reporting-org>
</iati-activity>
<iati-activity>
<iati-identifier>test_deleted_activity_2</iati-identifier>
<title>test_deleted_activity_2</title>
<reporting-org ref="GB-CHC-202918" type="21">Oxfam GB</reporting-org>
</iati-activity>
</iati-activities>
"""
)
crawler.parse_resource(resource)
db.session.commit()
db.session.query(Activity).update(
values={'last_change_datetime': datetime.datetime(2000, 1, 1)},
synchronize_session=False)
db.session.commit()
crawler.parse_resource(resource)
acts = db.session.query(Activity).all()
self.assertEquals(datetime.datetime(2000, 1, 1),
acts[0].last_change_datetime)
def test_last_changed_datetime(self):
resource = fac.ResourceFactory.create(url=u"http://test",
document="""
<iati-activities>
<iati-activity>
<iati-identifier>test_deleted_activity</iati-identifier>
<title>test_deleted_activity</title>
<reporting-org ref="GB-CHC-202918" type="21">Oxfam GB</reporting-org>
</iati-activity>
<iati-activity>
<iati-identifier>test_deleted_activity_2</iati-identifier>
<title>test_deleted_activity_2</title>
<reporting-org ref="GB-CHC-202918" type="21">Oxfam GB</reporting-org>
</iati-activity>
</iati-activities>
""")
crawler.parse_resource(resource)
db.session.commit()
db.session.query(Activity).update(
values={'last_change_datetime': datetime.datetime(2000, 1, 1)},
synchronize_session=False)
db.session.commit()
crawler.parse_resource(resource)
acts = db.session.query(Activity).all()
self.assertEquals(datetime.datetime(2000, 1, 1),
acts[0].last_change_datetime)
def test_deleted_activity_removal(self):
db.session.add(DeletedActivity(iati_identifier='test_deleted_activity',
deletion_date=datetime.datetime(2000, 1, 1)))
db.session.commit()
resource = fac.ResourceFactory.create(
url=u"http://test",
document="""
<iati-activities>
<iati-activity>
<iati-identifier>test_deleted_activity</iati-identifier>
<title>test_deleted_activity</title>
<reporting-org ref="GB-CHC-202918" type="21">Oxfam GB</reporting-org>
</iati-activity>
</iati-activities>
""",
)
self.assertIn(
"test_deleted_activity",
[da.iati_identifier for da in db.session.query(DeletedActivity).all()]
)
resource = crawler.parse_resource(resource)
db.session.commit()
self.assertNotIn(
"test_deleted_activity",
[da.iati_identifier for da in DeletedActivity.query.all()]
)
def test_document_metadata(self):
res = fac.ResourceFactory.create(
url=u"http://res2",
document=open(fixture_filename("complex_example_dfid.xml")).read())
result = crawler.parse_resource(res)
self.assertEquals("1.00", result.version)
def test_deleted_activity_removal(self):
db.session.add(
DeletedActivity(iati_identifier='test_deleted_activity',
deletion_date=datetime.datetime(2000, 1, 1)))
db.session.commit()
resource = fac.ResourceFactory.create(
url=u"http://test",
document="""
<iati-activities>
<iati-activity>
<iati-identifier>test_deleted_activity</iati-identifier>
<title>test_deleted_activity</title>
<reporting-org ref="GB-CHC-202918" type="21">Oxfam GB</reporting-org>
</iati-activity>
</iati-activities>
""",
)
self.assertIn("test_deleted_activity", [
da.iati_identifier
for da in db.session.query(DeletedActivity).all()
])
resource = crawler.parse_resource(resource)
db.session.commit()
self.assertNotIn(
"test_deleted_activity",
[da.iati_identifier for da in DeletedActivity.query.all()])
def test_document_metadata(self):
res = fac.ResourceFactory.create(
url=u"http://res2",
document=open(fixture_filename("complex_example_dfid.xml")).read()
)
result = crawler.parse_resource(res)
self.assertEquals("1.00", result.version)
def test_parse_resource_succ(self):
resource = Resource(document="<iati-activities />")
resource = crawler.parse_resource(resource)
self.assertEquals([], resource.activities)
self.assertEquals(None, resource.last_parse_error)
self.assertAlmostEquals(
resource.last_parsed,
datetime.datetime.utcnow(),
delta=datetime.timedelta(seconds=5))
def test_parse_resource_succ(self):
resource = Resource(document="<iati-activities />", url="http://foo")
resource = crawler.parse_resource(resource)
self.assertEquals([], resource.activities)
self.assertEquals(None, resource.last_parse_error)
now = datetime.datetime.utcnow()
self.assertAlmostEquals(resource.last_parsed,
now,
delta=datetime.timedelta(seconds=15))
def test_parse_resource_succ(self):
resource = Resource(document="<iati-activities />", url="http://foo")
resource = crawler.parse_resource(resource)
self.assertEquals([], list(resource.activities))
self.assertEquals(None, resource.last_parse_error)
now = datetime.datetime.utcnow()
self.assertAlmostEquals(
resource.last_parsed,
now,
delta=datetime.timedelta(seconds=15))
def test_parse_resource_succ_replaces_activities(self):
# what's in the db before the resource is updated
act = fac.ActivityFactory.build(iati_identifier="deleted_activity")
resource = fac.ResourceFactory.create(url=u"http://test",
activities=[act])
# the updated resource (will remove the activities)
resource.document = "<iati-activities />"
resource = crawler.parse_resource(resource)
db.session.commit()
self.assertEquals(None, Activity.query.get(act.iati_identifier))
self.assertIn(
"deleted_activity",
[da.iati_identifier for da in DeletedActivity.query.all()])
def test_parse_resource_succ_replaces_activities(self):
# what's in the db before the resource is updated
act = fac.ActivityFactory.build()
fac.ResourceFactory.create(
url=u"http://test",
activities=[act]
)
# the updated resource (will remove the activities)
resource = Resource(
url=u"http://test",
document="<iati-activities />"
)
resource = crawler.parse_resource(resource)
self.assertEquals(None, Activity.query.get(act.iati_identifier))
def test_parse_resource_succ_replaces_activities(self):
# what's in the db before the resource is updated
act = fac.ActivityFactory.build(iati_identifier="deleted_activity")
resource = fac.ResourceFactory.create(
url=u"http://test",
activities=[act]
)
# the updated resource (will remove the activities)
resource.document="<iati-activities />"
resource = crawler.parse_resource(resource)
db.session.commit()
self.assertEquals(None, Activity.query.get(act.iati_identifier))
self.assertIn(
"deleted_activity",
[da.iati_identifier for da in DeletedActivity.query.all()]
)
def test_parse_resource_fail(self):
resource = Resource(document="")
with self.assertRaises(parse.ParserError):
resource = crawler.parse_resource(resource)
self.assertEquals(None, resource.last_parsed)
def test_parse_resource_fail(self):
resource = Resource(document="")
with self.assertRaises(parse.ParserError):
resource = crawler.parse_resource(resource)
self.assertEquals(None, resource.last_parsed)