def setup(self):
"""
Create users and organizations
"""
reset_db()
search.clear()
self.sysadmin = factories.Sysadmin()
self.test_member_1 = factories.User()
self.test_member_2 = factories.User()
self.test_editor = factories.User()
users = [{
'name': self.test_member_1['name'],
'capacity': 'member'
}, {
'name': self.test_member_2['name'],
'capacity': 'member'
}, {
'name': self.test_editor['name'],
'capacity': 'editor'
}]
self.test_org_1 = factories.Organization(users=users,
name="testjournal")
self.test_org_2 = factories.Organization()
self.test_package = self._dara_testpackage()
def clear(dataset_name):
from ckan.lib.search import clear, clear_all
if dataset_name:
clear(dataset_name)
else:
clear_all()
def setup_class(cls):
search.clear()
CreateTestData.create()
cls.testsysadmin = model.User.by_name(u'testsysadmin')
cls.comment = u'Comment umlaut: \xfc.'
cls.user_name = u'annafan' # created in CreateTestData
cls.init_extra_environ(cls.user_name)
def setup_class(self):
try:
search.clear()
setup_test_search_index()
super(OnsLoaderBase, self).setup_class()
# make annafan a sysadmin to allow package creation
rev = model.repo.new_revision()
user = model.User.by_name(u'annafan')
model.add_user_to_role(user, model.Role.ADMIN, model.System())
model.repo.commit_and_remove()
publist = [g.name for g in model.Session.query(model.Group).all()]
# create test publishers
rev = model.repo.new_revision()
for name, title in publishers.items():
if not name in publist:
model.Session.add(model.Group(name=unicode(name), title=title, type='publisher'))
model.repo.commit_and_remove()
except Exception, e:
# ensure that mock_drupal is destroyed
print e
MockDrupalCase.teardown_class()
#model.repo.rebuild_db()
raise
def command(self):
self._load_config()
from ckan.lib.search import rebuild, check, show, clear
if not self.args:
# default to printing help
print self.usage
return
cmd = self.args[0]
if cmd == 'rebuild':
if len(self.args) > 1:
rebuild(self.args[1])
else:
rebuild()
elif cmd == 'check':
check()
elif cmd == 'show':
if not len(self.args) == 2:
import pdb; pdb.set_trace()
self.args
show(self.args[1])
elif cmd == 'clear':
clear()
else:
print 'Command %s not recognized' % cmd
def setup_class(self):
try:
search.clear()
setup_test_search_index()
super(OnsLoaderBase, self).setup_class()
# make annafan a sysadmin to allow package creation
rev = model.repo.new_revision()
user = model.User.by_name(u'annafan')
model.add_user_to_role(user, model.Role.ADMIN, model.System())
model.repo.commit_and_remove()
publist = [g.name for g in model.Session.query(model.Group).all()]
# create test publishers
rev = model.repo.new_revision()
for name, title in publishers.items():
if not name in publist:
model.Session.add(model.Group(name=unicode(name), title=title, type='publisher'))
model.repo.commit_and_remove()
except Exception, e:
# ensure that mock_drupal is destroyed
print e
MockDrupalCase.teardown_class()
#model.repo.rebuild_db()
raise
def setup_test_search_index():
from ckan import plugins
if not is_search_supported():
raise SkipTest("Search not supported")
search.clear()
plugins.load("synchronous_search")
def command(self):
self._load_config()
from ckan.lib.search import rebuild, check, show, clear
if not self.args:
# default to printing help
print self.usage
return
cmd = self.args[0]
if cmd == 'rebuild':
if len(self.args) > 1:
rebuild(self.args[1])
else:
rebuild()
elif cmd == 'check':
check()
elif cmd == 'show':
if not len(self.args) == 2:
import pdb
pdb.set_trace()
self.args
show(self.args[1])
elif cmd == 'clear':
clear()
else:
print 'Command %s not recognized' % cmd
def setup_class(cls):
search.clear()
CreateTestData.create()
cls.testsysadmin = model.User.by_name(u"testsysadmin")
cls.comment = u"Comment umlaut: \xfc."
cls.user_name = u"annafan" # created in CreateTestData
cls.init_extra_environ(cls.user_name)
def clear(dataset_name: str):
from ckan.lib.search import clear, clear_all
if dataset_name:
clear(dataset_name)
else:
clear_all()
def setup(self):
import ckan.model as model
# Reset the db before each test method.
model.repo.rebuild_db()
# Clear the search index
search.clear()
def setup_class(cls):
search.clear()
cls.tsi = TestSearchIndexer()
DguCreateTestData.create_dgu_test_data()
DguCreateTestData.create_arbitrary({'name': 'latest',
'notes': '<b>Latest</b> dataset.',
'groups': ['national-health-service']})
cls.tsi.index()
def setup(self):
import ckan.model as model
# Reset the db before each test method.
model.repo.rebuild_db()
# Clear the search index
search.clear()
def setup_class(cls):
# p.load('metadata_fields')
# TestMetadataFields.get_app()
search.clear()
model.Session.remove()
ctd.CreateTestData.create()
# Need to invalidate the caches to make sure the new data is being served
caching.invalidate_group_caches()
def setup_class(self):
search.clear()
model.Session.remove()
CreateTestData.create()
# reduce extraneous logging
from ckan.lib import activity_streams_session_extension
activity_streams_session_extension.logger.level = 100
def setup_class(cls):
search.clear()
cls.tsi = TestSearchIndexer()
DguCreateTestData.create_dgu_test_data()
DguCreateTestData.create_arbitrary(
{"name": "latest", "notes": "<b>Latest</b> dataset.", "groups": ["national-health-service"]}
)
cls.tsi.index()
def setup_class(cls):
search.clear()
CreateTestData.create()
cls.sysadmin_user = model.User.get("testsysadmin")
cls.sysadmin_action = TestAppCKAN(cls.app, str(cls.sysadmin_user.apikey)).action
cls.example_pkg = [json.loads(j) for j in EXAMPLE_JSON_LINES.strip().split("\n")]
def setup_class(self):
search.clear()
model.Session.remove()
CreateTestData.create()
# reduce extraneous logging
from ckan.lib import activity_streams_session_extension
activity_streams_session_extension.logger.level = 100
def setup_class(cls):
search.clear()
cls.tsi = TestSearchIndexer()
DguCreateTestData.create_dgu_test_data()
DguCreateTestData.create_arbitrary({'name': 'latest',
'notes': '<b>Latest</b> dataset.',
'groups': ['national-health-service']})
cls.tsi.index()
def setup_class(cls):
cls.original_config = config.copy()
cls._load_plugins()
cls.app = _get_test_app()
search.clear()
helpers.reset_db()
cls._create_test_data()
def setup_class(cls):
search.clear()
CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.sysadmin_action = TestAppCKAN(cls.app,
str(cls.sysadmin_user.apikey)).action
cls.example_pkg = [
json.loads(j) for j in EXAMPLE_JSON_LINES.strip().split('\n')
]
def setup_class(cls):
search.clear()
CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.normal_user = model.User.get('annafan')
cls.publisher_user = model.User.get('russianfan')
cls.sysadmin_action = TestAppCKAN(cls.app,
str(cls.sysadmin_user.apikey)).action
cls.normal_action = TestAppCKAN(cls.app,
str(cls.normal_user.apikey)).action
cls.publisher_action = TestAppCKAN(
cls.app, str(cls.publisher_user.apikey)).action
cls.action = TestAppCKAN(cls.app).action
cls.sysadmin_action.organization_member_create(username='******',
id='nrcan-rncan',
role='editor')
cls.sysadmin_action.organization_member_create(username='******',
id='tb-ct',
role='editor')
cls.incomplete_pkg = {
'title':
u'A Novel By Tolstoy',
'license_id':
'ca-ogl-lgo',
'resources': [{
'name': u'Full text.',
'name_fra': u'Full text.',
'format': u'TXT',
'url': u'http://www.annakarenina.com/download/',
'size': 42,
'resource_type': 'file',
'language': 'zxx; CAN',
}],
}
cls.override_possible_pkg = dict(cls.incomplete_pkg,
owner_org='nrcan-rncan')
cls.complete_pkg = dict(
cls.override_possible_pkg,
catalog_type=u'Data | Données',
title_fra=u'Un novel par Tolstoy',
maintenance_and_update_frequency=u'As Needed | Au besoin',
notes=u'...',
notes_fra=u'...',
subject=[u'Persons Personnes'],
date_published=u'2013-01-01',
keywords=u'book',
keywords_fra=u'livre')
def test_purged_dataset_is_not_in_search_results(self):
search.clear()
dataset = factories.Dataset()
def get_search_results():
results = helpers.call_action('package_search',
q=dataset['title'])['results']
return [d['name'] for d in results]
assert_equals(get_search_results(), [dataset['name']])
helpers.call_action('dataset_purge', id=dataset['name'])
assert_equals(get_search_results(), [])
def test_purged_dataset_is_not_in_search_results(self):
search.clear()
dataset = factories.Dataset()
def get_search_results():
results = helpers.call_action("package_search", q=dataset["title"])["results"]
return [d["name"] for d in results]
assert_equals(get_search_results(), [dataset["name"]])
helpers.call_action("dataset_purge", id=dataset["name"])
assert_equals(get_search_results(), [])
def test_purged_org_is_not_in_search_results_for_its_ex_dataset(self):
search.clear()
org = factories.Organization()
dataset = factories.Dataset(owner_org=org['id'])
def get_search_result_owner_org():
results = helpers.call_action('package_search',
q=dataset['title'])['results']
return results[0]['owner_org']
assert_equals(get_search_result_owner_org(), org['id'])
helpers.call_action('organization_purge', id=org['name'])
assert_equals(get_search_result_owner_org(), None)
def test_purged_group_is_not_in_search_results_for_its_ex_dataset(self):
search.clear()
group = factories.Group()
dataset = factories.Dataset(groups=[{'name': group['name']}])
def get_search_result_groups():
results = helpers.call_action('package_search',
q=dataset['title'])['results']
return [g['name'] for g in results[0]['groups']]
assert_equals(get_search_result_groups(), [group['name']])
helpers.call_action('group_purge', id=group['name'])
assert_equals(get_search_result_groups(), [])
def setup_class(cls):
'''
Set up test class
'''
super(TestSearchDataset, cls).setup_class()
search.clear()
data_dict = copy.deepcopy(cls.TEST_DATADICT) # Create public dataset
data_dict['private'] = False
# Create a dataset for this test class
output = call_action_api(cls.app, 'package_create', apikey=cls.user_sysadmin.apikey,
status=200, **data_dict)
cls.package_id = output.get('id')
def setup_class(cls):
search.clear()
CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.normal_user = model.User.get('annafan')
cls.publisher_user = model.User.get('russianfan')
cls.sysadmin_action = TestAppCKAN(cls.app,
str(cls.sysadmin_user.apikey)).action
cls.normal_action = TestAppCKAN(cls.app,
str(cls.normal_user.apikey)).action
cls.publisher_action = TestAppCKAN(cls.app,
str(cls.publisher_user.apikey)).action
cls.action = TestAppCKAN(cls.app).action
cls.sysadmin_action.organization_member_create(
username='******', id='nrcan-rncan', role='editor')
cls.sysadmin_action.organization_member_create(
username='******', id='tb-ct', role='editor')
cls.incomplete_pkg = {
'title': u'A Novel By Tolstoy',
'license_id': 'ca-ogl-lgo',
'resources': [{
'name': u'Full text.',
'name_fra': u'Full text.',
'format': u'TXT',
'url': u'http://www.annakarenina.com/download/',
'size': 42,
'resource_type': 'file',
'language': 'zxx; CAN',
}],
}
cls.override_possible_pkg = dict(cls.incomplete_pkg,
owner_org='nrcan-rncan')
cls.complete_pkg = dict(cls.override_possible_pkg,
catalog_type=u'Data | Données',
title_fra=u'Un novel par Tolstoy',
maintenance_and_update_frequency=u'As Needed | Au besoin',
notes=u'...',
notes_fra=u'...',
subject=[u'Persons Personnes'],
date_published=u'2013-01-01',
keywords=u'book',
keywords_fra=u'livre')
def setup_class(self):
search.clear()
setup_test_search_index()
super(OnsLoaderBase, self).setup_class()
# make annafan a sysadmin to allow package creation
rev = model.repo.new_revision()
user = model.User.by_name(u'annafan')
model.add_user_to_role(user, model.Role.ADMIN, model.System())
model.repo.commit_and_remove()
# create test publishers
rev = model.repo.new_revision()
for name, title in publishers.items():
model.Session.add(model.Group(name=unicode(name), title=title, type='publisher'))
model.repo.commit_and_remove()
def command(self):
# Avoids vdm logging warning
logging.basicConfig(level=logging.ERROR)
self._load_config()
from ckan import model
import ckan.lib.search as search
cmd = self.args[0]
if cmd == 'init':
model.repo.init_db()
if self.verbose:
print 'Initialising DB: SUCCESS'
elif cmd == 'clean' or cmd == 'drop':
model.repo.clean_db()
search.clear()
if self.verbose:
print 'Cleaning DB: SUCCESS'
elif cmd == 'upgrade':
if len(self.args) > 1:
model.repo.upgrade_db(self.args[1])
else:
model.repo.upgrade_db()
elif cmd == 'version':
self.version()
elif cmd == 'dump':
self.dump()
elif cmd == 'load':
self.load()
elif cmd == 'load-only':
self.load(only_load=True)
elif cmd == 'simple-dump-csv':
self.simple_dump_csv()
elif cmd == 'simple-dump-json':
self.simple_dump_json()
elif cmd == 'dump-rdf':
self.dump_rdf()
elif cmd == 'create-from-model':
model.repo.create_db()
if self.verbose:
print 'Creating DB: SUCCESS'
elif cmd == 'send-rdf':
self.send_rdf()
else:
print 'Command %s not recognized' % cmd
sys.exit(1)
def setup_class(self):
search.clear()
setup_test_search_index()
super(OnsLoaderBase, self).setup_class()
# make annafan a sysadmin to allow package creation
rev = model.repo.new_revision()
user = model.User.by_name(u'annafan')
model.add_user_to_role(user, model.Role.ADMIN, model.System())
model.repo.commit_and_remove()
# create test publishers
rev = model.repo.new_revision()
for name, title in publishers.items():
model.Session.add(
model.Group(name=unicode(name), title=title, type='publisher'))
model.repo.commit_and_remove()
def command(self):
# Avoids vdm logging warning
logging.basicConfig(level=logging.ERROR)
self._load_config()
from ckan import model
import ckan.lib.search as search
cmd = self.args[0]
if cmd == 'init':
model.repo.init_db()
if self.verbose:
print 'Initialising DB: SUCCESS'
elif cmd == 'clean' or cmd == 'drop':
model.repo.clean_db()
search.clear()
if self.verbose:
print 'Cleaning DB: SUCCESS'
elif cmd == 'upgrade':
if len(self.args) > 1:
model.repo.upgrade_db(self.args[1])
else:
model.repo.upgrade_db()
elif cmd == 'version':
self.version()
elif cmd == 'dump':
self.dump()
elif cmd == 'load':
self.load()
elif cmd == 'load-only':
self.load(only_load=True)
elif cmd == 'simple-dump-csv':
self.simple_dump_csv()
elif cmd == 'simple-dump-json':
self.simple_dump_json()
elif cmd == 'dump-rdf':
self.dump_rdf()
elif cmd == 'create-from-model':
model.repo.create_db()
if self.verbose:
print 'Creating DB: SUCCESS'
elif cmd == 'send-rdf':
self.send_rdf()
else:
print 'Command %s not recognized' % cmd
sys.exit(1)
def setup(self):
"""
Create users and organizations
"""
reset_db()
search.clear()
self.sysadmin = factories.Sysadmin()
self.test_member_1 = factories.User()
self.test_member_2 = factories.User()
self.test_editor = factories.User()
users = [{'name': self.test_member_1['name'], 'capacity': 'member'},
{'name': self.test_member_2['name'], 'capacity': 'member'},
{'name': self.test_editor['name'], 'capacity': 'editor'}]
self.test_org_1 = factories.Organization(users=users,
name="testjournal")
self.test_org_2 = factories.Organization()
self.test_package = self._dara_testpackage()
def setup_class(cls):
# clean the db so we can run these tests on their own
model.repo.rebuild_db()
search.clear()
CreateTestData.create()
cls.package_expected = {
u'author': None,
u'author_email': None,
u'creator_user_id': None,
'extras': [
{u'key': u'genre',
u'state': u'active',
u'value': 'romantic novel'},
{u'key': u'original media', u'state': u'active', u'value': u'book'}],
'groups': [{
u'name': u'david',
u'capacity': u'public',
u'image_url': u'',
u'image_display_url': u'',
u'description': u'These are books that David likes.',
u'display_name': u"Dave's books",
u'type': u'group',
u'state': u'active',
u'is_organization': False,
u'title': u"Dave's books",
u"approval_status": u"approved"},
{
u'name': u'roger',
u'capacity': u'public',
u'description': u'Roger likes these books.',
u'image_url': u'',
'image_display_url': u'',
'display_name': u"Roger's books",
u'type': u'group',
u'state': u'active',
u'is_organization': False,
u'title': u"Roger's books",
u"approval_status": u"approved"}],
'isopen': True,
u'license_id': u'other-open',
'license_title': u'Other (Open)',
'organization': None,
u'owner_org': None,
u'maintainer': None,
u'maintainer_email': None,
u'name': u'annakarenina',
u'notes': u'Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n\nNeeds escaping:\nleft arrow <\n\n<http://ckan.net/>\n\n',
'num_resources': 2,
'num_tags': 3,
u'private': False,
'relationships_as_object': [],
'relationships_as_subject': [],
'resources': [{u'alt_url': u'alt123',
u'cache_last_updated': None,
u'cache_url': None,
u'description': u'Full text. Needs escaping: " Umlaut: \xfc',
u'format': u'plain text',
u'hash': u'abc123',
u'last_modified': None,
u'mimetype': None,
u'mimetype_inner': None,
u'name': None,
u'position': 0,
u'resource_type': None,
u'size': None,
u'size_extra': u'123',
u'url_type': None,
u'state': u'active',
u'url': u'http://www.annakarenina.com/download/x=1&y=2',
u'webstore_last_updated': None,
u'webstore_url': None},
{u'alt_url': u'alt345',
u'cache_last_updated': None,
u'cache_url': None,
u'description': u'Index of the novel',
u'format': u'JSON',
u'hash': u'def456',
u'last_modified': None,
u'mimetype': None,
u'mimetype_inner': None,
u'name': None,
u'position': 1,
u'resource_type': None,
u'url_type': None,
u'size': None,
u'size_extra': u'345',
u'state': u'active',
u'url': u'http://www.annakarenina.com/index.json',
u'webstore_last_updated': None,
u'webstore_url': None}],
u'state': u'active',
'tags': [{u'name': u'Flexible \u30a1',
'display_name': u'Flexible \u30a1',
u'state': u'active'},
{'display_name': u'russian',
u'name': u'russian',
u'state': u'active'},
{'display_name': u'tolstoy',
u'name': u'tolstoy',
u'state': u'active'}],
u'title': u'A Novel By Tolstoy',
u'type': u'dataset',
u'url': u'http://www.annakarenina.com',
u'version': u'0.7a',
}
def teardown_class(cls):
search.clear()
model.repo.rebuild_db()
def setup(self):
helpers.reset_db()
search.clear()
def teardown_class(self):
model.repo.rebuild_db()
model.Session.remove()
search.clear()
def setup_test_search_index():
#from ckan import plugins
if not is_search_supported():
raise SkipTest("Search not supported")
search.clear()
def setup(self):
helpers.reset_db()
search.clear()
def setup_class(cls):
search.clear()
CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.normal_user = model.User.get('annafan')
CreateTestData.make_some_vocab_tags()
def teardown(self):
search.clear()
def setup_class(self):
search.clear()
model.Session.remove()
CreateTestData.create()
def teardown_class(cls):
helpers.reset_db()
search.clear()
def db_solr_sync(self):
print str(
datetime.datetime.now()) + ' Entering Database Solr Sync function.'
url = config.get(
'solr_url'
) + "/select?q=*%3A*&sort=id+asc&fl=id%2Cmetadata_modified&wt=json&indent=true"
response = get_response(url)
if (response != 'error'):
print str(datetime.datetime.now()
) + ' Deleting records from solr_pkg_ids.'
sql = '''delete from solr_pkg_ids'''
model.Session.execute(sql)
model.Session.commit()
f = response.read()
data = json.loads(f)
rows = data.get('response').get('numFound')
start = 0
chunk_size = 1000
print str(datetime.datetime.now()
) + ' Starting insertion of records in solr_pkg_ids .'
for x in range(0, int(math.ceil(rows / chunk_size)) + 1):
if (x == 0):
start = 0
print str(datetime.datetime.now()
) + ' Fetching ' + url + "&rows=" + str(
chunk_size) + "&start=" + str(start)
response = get_response(url + "&rows=" + str(chunk_size) +
"&start=" + str(start))
f = response.read()
data = json.loads(f)
results = data.get('response').get('docs')
print str(datetime.datetime.now()) + ' Inserting ' + str(
start) + ' - ' + str(start + int(
data.get('responseHeader').get('params').get('rows')) -
1) + ' of ' + str(rows)
for x in range(0, len(results)):
sql = '''select count(id) as count from package where id = :pkg_id;'''
q = model.Session.execute(sql,
{'pkg_id': results[x]['id']})
for row in q:
if (row['count'] == 0):
sql = '''insert into solr_pkg_ids (pkg_id, action) values (:pkg_id, :action);'''
model.Session.execute(sql, {
'pkg_id': results[x]['id'],
'action': 'notfound'
})
model.Session.commit()
else:
pkg_dict = logic.get_action('package_show')(
{
'model': model,
'ignore_auth': True,
'validate': False
}, {
'id': results[x]['id']
})
if (str(results[x]['metadata_modified'])[:19] !=
pkg_dict['metadata_modified'][:19]):
print str(
datetime.datetime.now()
) + ' Action Type : outsync for Package Id: ' + results[
x]['id']
print ' ' * 26 + ' Modified Date from Solr: ' + str(
results[x]['metadata_modified'])
print ' ' * 26 + ' Modified Date from Db: ' + pkg_dict[
'metadata_modified']
sql = '''insert into solr_pkg_ids (pkg_id, action) values (:pkg_id, :action);'''
model.Session.execute(
sql, {
'pkg_id': results[x]['id'],
'action': 'outsync'
})
model.Session.commit()
else:
sql = '''insert into solr_pkg_ids (pkg_id, action) values (:pkg_id, :action);'''
model.Session.execute(
sql, {
'pkg_id': results[x]['id'],
'action': 'insync'
})
model.Session.commit()
start = int(
data.get('responseHeader').get('params').get(
'start')) + chunk_size
print str(
datetime.datetime.now()) + ' Starting Database to Solr Sync'
#sql = '''Select id from package where id not in (select pkg_id from solr_pkg_ids); '''
sql = '''Select p.id as pkg_id from package p
left join solr_pkg_ids sp on sp.pkg_id = p.id
where sp.pkg_id is null; '''
q = model.Session.execute(sql)
pkg_ids = set()
for row in q:
pkg_ids.add(row['pkg_id'])
for pkg_id in pkg_ids:
try:
print str(
datetime.datetime.now()) + ' Building Id: ' + pkg_id
search.rebuild(pkg_id)
except ckan.logic.NotFound:
print "Error: Not Found."
except KeyboardInterrupt:
print "Stopped."
return
except:
raise
sql = '''Select pkg_id from solr_pkg_ids where action = 'outsync'; '''
q = model.Session.execute(sql)
pkg_ids = set()
for row in q:
pkg_ids.add(row['pkg_id'])
for pkg_id in pkg_ids:
try:
print str(
datetime.datetime.now()) + ' Rebuilding Id: ' + pkg_id
search.rebuild(pkg_id)
except ckan.logic.NotFound:
print "Error: Not Found."
except KeyboardInterrupt:
print "Stopped."
return
except:
raise
print str(
datetime.datetime.now()) + ' Starting Solr to Database Sync'
sql = '''Select pkg_id from solr_pkg_ids where action = 'notfound'; '''
q = model.Session.execute(sql)
pkg_ids = set()
for row in q:
pkg_ids.add(row['pkg_id'])
for pkg_id in pkg_ids:
try:
search.clear(pkg_id)
except ckan.logic.NotFound:
print "Error: Not Found."
except KeyboardInterrupt:
print "Stopped."
return
except:
raise
print str(datetime.datetime.now()) + " All Sync Done."
def setup_class(cls):
search.clear()
CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.normal_user = model.User.get('annafan')
CreateTestData.make_some_vocab_tags()
def setup_class(self):
search.clear()
super(OnsLoaderBase, self).setup_class()
def format_mapping(self):
try:
tk.check_access('sysadmin', {'user': g.user, model: model})
except tk.NotAuthorized:
return tk.abort(403)
if request.method == 'POST':
old = request.POST.get('from')
new = request.POST.get('to')
if old and new:
ids = set()
res_query = model.Session.query(model.Resource).filter_by(
format=old, state='active'
)
for res in res_query:
ids.add(res.package_id)
res_query.update({'format': new})
model.Session.commit()
for id in ids:
clear(id)
rebuild(id, defer_commit=True)
commit()
tk.h.flash_success(
'Updated. Records changed: {}'.format(len(ids))
)
return tk.redirect_to('format_mapping')
defined = set(
map(lambda (_1, fmt, _3): fmt,
h.resource_formats().values())
)
db_formats = model.Session.query(
model.Resource.format, func.count(model.Resource.id),
func.count(model.PackageExtra.value)
).outerjoin(
model.PackageExtra,
(model.Resource.package_id == model.PackageExtra.package_id)
& ((model.PackageExtra.key == 'harvest_portal')
| (model.PackageExtra.key.is_(None)))
).group_by(model.Resource.format).filter(
model.Resource.format != '', model.Resource.state == 'active'
)
db_formats = db_formats.all()
format_types = {
f: {
True: 'Partially external',
e == 0: 'Local',
t - e == 0: 'External'
}[True]
for (f, t, e) in db_formats
}
used = set(format_types)
undefined = used - defined
extra_vars = {
'undefined': undefined,
'defined': defined,
'format_types': format_types
}
return tk.render('admin/format_mapping.html', extra_vars)
def teardown(self):
helpers.reset_db()
search.clear()
def setup(self):
'''Reset the database and clear the search indexes.'''
reset_db()
search.clear()
def setup(self):
helpers.reset_db()
# Clear the search index
search.clear()
def setup_test_search_index():
from ckan import plugins
if not is_search_supported():
raise SkipTest("Search not supported")
search.clear()
plugins.load('synchronous_search')
def teardown(self):
search.clear()
def setup_class(cls):
# clean the db so we can run these tests on their own
model.repo.rebuild_db()
search.clear()
CreateTestData.create()
cls.package_expected = {
'author':
None,
'author_email':
None,
'extras': [{
'key': u'genre',
'state': u'active',
'value': 'romantic novel'
}, {
'key': u'original media',
'state': u'active',
'value': u'book'
}],
'groups': [{
'description': u'These are books that David likes.',
'name': u'david',
'capacity': 'public',
'image_url': u'',
'image_display_url': u'',
'display_name': u"Dave's books",
'type': u'group',
'state': u'active',
'is_organization': False,
'title': u"Dave's books",
"approval_status": u"approved"
}, {
'description': u'Roger likes these books.',
'name': u'roger',
'capacity': 'public',
'image_url': u'',
'image_display_url': u'',
'display_name': u"Roger's books",
'type': u'group',
'state': u'active',
'is_organization': False,
'title': u"Roger's books",
"approval_status": u"approved"
}],
'isopen':
True,
'license_id':
u'other-open',
'license_title':
u'Other (Open)',
'creator_user_id':
None,
'owner_org':
None,
'private':
False,
'organization':
None,
'maintainer':
None,
'maintainer_email':
None,
'type':
u'dataset',
'name':
u'annakarenina',
'notes':
u'Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n\nNeeds escaping:\nleft arrow <\n\n<http://ckan.net/>\n\n',
'relationships_as_object': [],
'relationships_as_subject': [],
'resources': [{
u'alt_url': u'alt123',
u'cache_last_updated': None,
u'cache_url': None,
u'description': u'Full text. Needs escaping: " Umlaut: \xfc',
u'format': u'plain text',
u'hash': u'abc123',
u'last_modified': None,
u'mimetype': None,
u'mimetype_inner': None,
u'name': None,
u'position': 0,
u'resource_type': None,
u'size': None,
u'size_extra': u'123',
'url_type': None,
u'state': u'active',
u'url': u'http://www.annakarenina.com/download/x=1&y=2',
u'webstore_last_updated': None,
u'webstore_url': None
}, {
u'alt_url': u'alt345',
u'cache_last_updated': None,
u'cache_url': None,
u'description': u'Index of the novel',
u'format': u'JSON',
u'hash': u'def456',
u'last_modified': None,
u'mimetype': None,
u'mimetype_inner': None,
u'name': None,
u'position': 1,
u'resource_type': None,
'url_type': None,
u'size': None,
u'size_extra': u'345',
u'state': u'active',
u'url': u'http://www.annakarenina.com/index.json',
u'webstore_last_updated': None,
u'webstore_url': None
}],
'state':
u'active',
'tags': [{
'name': u'Flexible \u30a1',
'display_name': u'Flexible \u30a1',
'state': u'active'
}, {
'name': u'russian',
'display_name': u'russian',
'state': u'active'
}, {
'name': u'tolstoy',
'display_name': u'tolstoy',
'state': u'active'
}],
'title':
u'A Novel By Tolstoy',
'url':
u'http://www.annakarenina.com',
'version':
u'0.7a',
'num_tags':
3,
'num_resources':
2,
}
def teardown_class(self):
model.repo.rebuild_db()
model.Session.remove()
search.clear()
def teardown_class(cls):
model.repo.rebuild_db()
search.clear()
def setup(self):
helpers.reset_db()
# Clear the search index
search.clear()
def db_solr_sync(self):
print str(datetime.datetime.now()) + ' Entering Database Solr Sync function.'
url = config.get('solr_url') + "/select?q=*%3A*&sort=id+asc&fl=id%2Cmetadata_modified&wt=json&indent=true"
response = get_response(url)
if (response != 'error'):
print str(datetime.datetime.now()) + ' Deleting records from solr_pkg_ids.'
sql = '''delete from solr_pkg_ids'''
model.Session.execute(sql)
model.Session.commit()
f = response.read()
data = json.loads(f)
rows = data.get('response').get('numFound')
start = 0
chunk_size = 1000
print str(datetime.datetime.now()) + ' Starting insertion of records in solr_pkg_ids .'
for x in range(0, int(math.ceil(rows/chunk_size))+1):
if(x == 0):
start = 0
print str(datetime.datetime.now()) + ' Fetching ' + url + "&rows=" + str(chunk_size) + "&start=" + str(start)
response = get_response(url + "&rows=" + str(chunk_size) + "&start=" + str(start))
f = response.read()
data = json.loads(f)
results = data.get('response').get('docs')
print str(datetime.datetime.now()) + ' Inserting ' + str(start) + ' - ' + str(start + int(data.get('responseHeader').get('params').get('rows')) - 1) + ' of ' + str(rows)
for x in range(0, len(results)):
sql = '''select count(id) as count from package where id = :pkg_id;'''
q = model.Session.execute(sql, {'pkg_id' : results[x]['id']})
for row in q:
if(row['count'] == 0):
sql = '''insert into solr_pkg_ids (pkg_id, action) values (:pkg_id, :action);'''
model.Session.execute(sql, {'pkg_id' : results[x]['id'], 'action' : 'notfound' })
model.Session.commit()
else:
pkg_dict = logic.get_action('package_show')(
{'model': model, 'ignore_auth': True, 'validate': False},
{'id': results[x]['id']})
if(str(results[x]['metadata_modified'])[:19] != pkg_dict['metadata_modified'][:19]):
print str(datetime.datetime.now()) + ' Action Type : outsync for Package Id: ' + results[x]['id']
print ' ' * 26 + ' Modified Date from Solr: ' + str(results[x]['metadata_modified'])
print ' ' * 26 + ' Modified Date from Db: ' + pkg_dict['metadata_modified']
sql = '''insert into solr_pkg_ids (pkg_id, action) values (:pkg_id, :action);'''
model.Session.execute(sql, {'pkg_id' : results[x]['id'], 'action' : 'outsync' })
model.Session.commit()
else:
sql = '''insert into solr_pkg_ids (pkg_id, action) values (:pkg_id, :action);'''
model.Session.execute(sql, {'pkg_id' : results[x]['id'], 'action' : 'insync' })
model.Session.commit()
start = int(data.get('responseHeader').get('params').get('start')) + chunk_size
print str(datetime.datetime.now()) + ' Starting Database to Solr Sync'
#sql = '''Select id from package where id not in (select pkg_id from solr_pkg_ids); '''
sql = '''Select p.id as pkg_id from package p
left join solr_pkg_ids sp on sp.pkg_id = p.id
where sp.pkg_id is null; '''
q = model.Session.execute(sql)
pkg_ids = set()
for row in q:
pkg_ids.add(row['pkg_id'])
for pkg_id in pkg_ids:
try:
print str(datetime.datetime.now()) + ' Building Id: ' + pkg_id
search.rebuild(pkg_id)
except ckan.logic.NotFound:
print "Error: Not Found."
except KeyboardInterrupt:
print "Stopped."
return
except:
raise
sql = '''Select pkg_id from solr_pkg_ids where action = 'outsync'; '''
q = model.Session.execute(sql)
pkg_ids = set()
for row in q:
pkg_ids.add(row['pkg_id'])
for pkg_id in pkg_ids:
try:
print str(datetime.datetime.now()) + ' Rebuilding Id: ' + pkg_id
search.rebuild(pkg_id)
except ckan.logic.NotFound:
print "Error: Not Found."
except KeyboardInterrupt:
print "Stopped."
return
except:
raise
print str(datetime.datetime.now()) + ' Starting Solr to Database Sync'
sql = '''Select pkg_id from solr_pkg_ids where action = 'notfound'; '''
q = model.Session.execute(sql)
pkg_ids = set()
for row in q:
pkg_ids.add(row['pkg_id'])
for pkg_id in pkg_ids:
try:
search.clear(pkg_id)
except ckan.logic.NotFound:
print "Error: Not Found."
except KeyboardInterrupt:
print "Stopped."
return
except:
raise
print str(datetime.datetime.now()) + " All Sync Done."
def clear(self):
from ckan.lib.search import clear
package_id =self.args[1] if len(self.args) > 1 else None
clear(package_id)