def test_gather(self):
source = HarvestSource(url="http://localhost/test_cmdi", type="cmdi")
source.save()
job = HarvestJob(source=source)
job.save()
self.harvester.client = _FakeClient()
self.harvester.gather_stage(job)
def harvest_sources(self):
ddi = HarvestSource(url='http://www.fsd.uta.fi/fi/aineistot/luettelo/fsd-ddi-records-uris-fi.txt',
type='DDI')
ddi.save()
oai = HarvestSource(url='http://helda.helsinki.fi/oai/request',
type='OAI-PMH')
oai.save()
def harvest_source_create(context, data_dict):
log.info('Creating harvest source: %r', data_dict)
check_access('harvest_source_create', context, data_dict)
model = context['model']
session = context['session']
schema = context.get('schema') or default_harvest_source_schema()
data, errors = validate(data_dict, schema)
if errors:
session.rollback()
log.warn('Harvest source does not validate: %r', errors)
raise ValidationError(errors, _error_summary(errors))
source = HarvestSource()
source.url = data['url'].strip()
source.type = data['type']
opt = [
'active', 'title', 'description', 'user_id', 'publisher_id', 'config'
]
for o in opt:
if o in data and data[o] is not None:
source.__setattr__(o, data[o])
if 'active' in data_dict:
source.active = data['active']
source.save()
log.info('Harvest source created: %s', source.id)
return harvest_source_dictize(source, context)
def test_gather(self):
source = HarvestSource(url="http://localhost/test_cmdi", type="cmdi")
source.save()
job = HarvestJob(source=source)
job.save()
self.harvester.client = _FakeClient()
self.harvester.gather_stage(job)
def harvest_source_create(context,data_dict):
log.info('Creating harvest source: %r', data_dict)
check_access('harvest_source_create',context,data_dict)
model = context['model']
session = context['session']
schema = context.get('schema') or default_harvest_source_schema()
data, errors = validate(data_dict, schema)
if errors:
session.rollback()
log.warn('Harvest source does not validate: %r', errors)
raise ValidationError(errors,_error_summary(errors))
source = HarvestSource()
source.url = data['url'].strip()
source.type = data['type']
opt = ['active','title','description','user_id','publisher_id','config']
for o in opt:
if o in data and data[o] is not None:
source.__setattr__(o,data[o])
if 'active' in data_dict:
source.active = data['active']
source.save()
log.info('Harvest source created: %s', source.id)
return harvest_source_dictize(source,context)
def test_import(self):
source = HarvestSource(url="http://localhost/test_cmdi", type="cmdi")
source.save()
job = HarvestJob(source=source)
job.save()
harvest_object = self._run_import("cmdi_1.xml", job)
self.assertEquals(len(harvest_object.errors), 0, u"\n".join(unicode(error.message) for error in (harvest_object.errors or [])))
package = get_action('package_show')({'user': '******'}, {'id': 'urn-nbn-fi-lb-20140730180'})
self.assertEquals(package.get('id', None), 'http://urn.fi/urn:nbn:fi:lb-20140730180')
self.assertEquals(package.get('name', None), 'urn-nbn-fi-lb-20140730180')
self.assertEquals(package.get('notes', None), u'{"eng": "Test description"}')
self.assertEquals(package.get('version', None), '2012-09-07')
self.assertEquals(package.get('title', []), '{"eng": "Longi Corpus"}')
self.assertEquals(package.get('license_id', None), 'undernegotiation')
provider = config['ckan.site_url']
expected_pid = {u'id': u'http://islrn.org/resources/248-895-085-557-0',
u'provider': provider,
u'type': u'metadata'}
self.assertTrue(expected_pid in package.get('pids'))
model.Session.flush()
harvest_object = self._run_import("cmdi_2.xml", job)
self.assertEquals(len(harvest_object.errors), 0, u"\n".join(unicode(error.message) for error in (harvest_object.errors or [])))
package = get_action('package_show')({'user': '******'}, {'id': 'urn-nbn-fi-lb-20140730186'})
self.assertEquals(package['temporal_coverage_begin'], '1880')
self.assertEquals(package['temporal_coverage_end'], '1939')
self.assertEquals(package.get('license_id', None), 'other')
# Delete package
harvest_object = HarvestObject()
harvest_object.content = None
harvest_object.id = "test-cmdi-delete"
harvest_object.guid = "test-cmdi-delete"
harvest_object.source = job.source
harvest_object.harvest_source_id = None
harvest_object.job = job
harvest_object.package_id = package.get('id')
harvest_object.report_status = "deleted"
harvest_object.save()
self.harvester.import_stage(harvest_object)
model.Session.flush()
self.assertEquals(model.Package.get(package['id']).state, 'deleted')
def setup_class(cls):
# Create package and its harvest object
CreateTestData.create()
harvest_setup()
source = HarvestSource(url=u'http://test-source.org', type='test')
source.save()
job = HarvestJob(source=source)
job.save()
ho = HarvestObject(package=model.Package.by_name(u'annakarenina'),
job=job,
guid=u'test-guid',
content=u'<xml>test content</xml>')
ho.save()
# Save a reference to the harvest object in the package
rev = model.repo.new_revision()
pkg = model.Package.by_name(u'annakarenina')
pkg.extras['harvest_object_id'] = ho.id
pkg.save()
model.repo.commit_and_remove()
def setup_class(cls):
# Create package and its harvest object
CreateTestData.create()
harvest_setup()
source = HarvestSource(url=u'http://test-source.org',type='test')
source.save()
job = HarvestJob(source=source)
job.save()
ho = HarvestObject(package=model.Package.by_name(u'annakarenina'),
job=job,
guid=u'test-guid',
content=u'<xml>test content</xml>')
ho.save()
# Save a reference to the harvest object in the package
rev = model.repo.new_revision()
pkg = model.Package.by_name(u'annakarenina')
pkg.extras['harvest_object_id'] = ho.id
pkg.save()
model.repo.commit_and_remove()
def test_import(self):
source = HarvestSource(url="http://localhost/test_cmdi", type="cmdi")
source.save()
job = HarvestJob(source=source)
job.save()
harvest_object = self._run_import("cmdi_1.xml", job)
package_id = json.loads(harvest_object.content)['unified']['id']
self.assertEquals(
len(harvest_object.errors), 0, u"\n".join(
unicode(error.message)
for error in (harvest_object.errors or [])))
package = get_action('package_show')({
'user': '******'
}, {
'id': package_id
})
self.assertEquals(package.get('name', None),
utils.pid_to_name(package.get('id', None)))
self.assertEquals(utils.get_primary_pid(package),
u'http://urn.fi/urn:nbn:fi:lb-20140730180')
self.assertEquals(package.get('notes', None),
u'{"eng": "Test description"}')
self.assertEquals(package.get('version', None), '2012-09-07')
self.assertEquals(package.get('title', []), '{"eng": "Longi Corpus"}')
self.assertEquals(package.get('license_id', None), 'undernegotiation')
provider = config['ckan.site_url']
expected_pid = {
u'id': u'http://islrn.org/resources/248-895-085-557-0',
u'provider': provider,
u'type': u'relation',
u'relation': u'generalRelation'
}
self.assertTrue(expected_pid not in package.get('pids'))
model.Session.flush()
harvest_object = self._run_import("cmdi_2.xml", job)
package_id = json.loads(harvest_object.content)['unified']['id']
self.assertEquals(
len(harvest_object.errors), 0, u"\n".join(
unicode(error.message)
for error in (harvest_object.errors or [])))
package = get_action('package_show')({
'user': '******'
}, {
'id': package_id
})
self.assertEquals(package['temporal_coverage_begin'], '1880')
self.assertEquals(package['temporal_coverage_end'], '1939')
self.assertEquals(package.get('license_id', None), 'other')
# Delete package
harvest_object = HarvestObject()
harvest_object.content = None
harvest_object.id = "test-cmdi-delete"
harvest_object.guid = "test-cmdi-delete"
harvest_object.source = job.source
harvest_object.harvest_source_id = None
harvest_object.job = job
harvest_object.package_id = package.get('id')
harvest_object.report_status = "deleted"
harvest_object.save()
self.harvester.import_stage(harvest_object)
model.Session.flush()
self.assertEquals(model.Package.get(package['id']).state, 'deleted')
def harvest_sources(self):
ddi = HarvestSource(url='http://www.fsd.uta.fi/fi/aineistot/luettelo/fsd-ddi-records-uris-fi.txt',
type='DDI')
ddi.save()
def harvest_sources(self):
ddi = HarvestSource(
url=
'http://www.fsd.uta.fi/fi/aineistot/luettelo/fsd-ddi-records-uris-fi.txt',
type='DDI')
ddi.save()
