def gather_stage(self, harvest_job):
# make sure we have all the right organizations
url = harvest_job.source.url
session = requests.Session()
r = session.get("{}/api/action/organization_list".format(url))
if r.json()["success"]:
remote_organizations = r.json()['result']
local_organizations = model.Group.all("organization")
local_organization_names = [
org.name for org in local_organizations
]
for remote_org in remote_organizations:
if remote_org not in local_organization_names:
context = {
'model': model,
'session': Session,
'user': self._get_user_name(),
'ignore_auth': True,
}
session = requests.Session()
r = session.get(
"{}/api/action/organization_show?id={}".format(
url, remote_org))
if r.json()["success"]:
remote_organization = r.json()['result']
new_package = p.toolkit.get_action(
"organization_create")(context,
remote_organization)
return CKANHarvester.gather_stage(self, harvest_job)
def test_gather_normal(self):
source = HarvestSourceObj(url='http://localhost:%s/' % mock_ckan.PORT)
job = HarvestJobObj(source=source)
harvester = CKANHarvester()
obj_ids = harvester.gather_stage(job)
assert_equal(type(obj_ids), list)
assert_equal(len(obj_ids), len(mock_ckan.DATASETS))
harvest_object = harvest_model.HarvestObject.get(obj_ids[0])
assert_equal(harvest_object.guid, mock_ckan.DATASETS[0]['id'])
def test_gather_normal(self):
source = HarvestSourceObj(url='http://localhost:%s/' % mock_ckan.PORT)
job = HarvestJobObj(source=source)
harvester = CKANHarvester()
obj_ids = harvester.gather_stage(job)
assert_equal(type(obj_ids), list)
assert_equal(len(obj_ids), len(mock_ckan.DATASETS))
harvest_object = harvest_model.HarvestObject.get(obj_ids[0])
assert_equal(harvest_object.guid, mock_ckan.DATASETS[0]['id'])
def test_gather_normal(self):
source = HarvestSourceObj(url='http://localhost:%s/' % mock_ckan.PORT)
job = HarvestJobObj(source=source)
harvester = CKANHarvester()
obj_ids = harvester.gather_stage(job)
assert job.gather_errors == []
assert type(obj_ids) == list
assert len(obj_ids) == len(mock_ckan.DATASETS)
harvest_object = harvest_model.HarvestObject.get(obj_ids[0])
assert harvest_object.guid == mock_ckan.DATASETS[0]['id']
assert json.loads(harvest_object.content) == mock_ckan.DATASETS[0]