def setup_class(cls):
cls.app = helpers._get_test_app()
super(TestDatastoreInsertLegacyTests, cls).setup_class()
ctd.CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.normal_user = model.User.get('annafan')
set_url_type(
model.Package.get('annakarenina').resources, cls.sysadmin_user)
resource = model.Package.get('annakarenina').resources[0]
cls.data = {
'resource_id': resource.id,
'fields': [{'id': u'b\xfck', 'type': 'text'},
{'id': 'author', 'type': 'text'},
{'id': 'nested', 'type': 'json'},
{'id': 'characters', 'type': 'text[]'},
{'id': 'published'}],
'primary_key': u'b\xfck',
'records': [{u'b\xfck': 'annakarenina', 'author': 'tolstoy',
'published': '2005-03-01', 'nested': ['b', {'moo': 'moo'}]},
{u'b\xfck': 'warandpeace', 'author': 'tolstoy',
'nested': {'a':'b'}}
]
}
postparams = '%s=1' % json.dumps(cls.data)
auth = {'Authorization': str(cls.sysadmin_user.apikey)}
res = cls.app.post('/api/action/datastore_create', params=postparams,
extra_environ=auth)
res_dict = json.loads(res.body)
assert res_dict['success'] is True
engine = db.get_write_engine()
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
def setup_class(cls):
if not tests.is_datastore_supported():
raise nose.SkipTest("Datastore not supported")
plugin = p.load('datastore')
if plugin.legacy_mode:
# make sure we undo adding the plugin
p.unload('datastore')
raise nose.SkipTest("SQL tests are not supported in legacy mode")
ctd.CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.normal_user = model.User.get('annafan')
cls.dataset = model.Package.get('annakarenina')
resource = cls.dataset.resources[0]
cls.data = {
'resource_id': resource.id,
'force': True,
'aliases': 'books4',
'fields': [{'id': u'b\xfck', 'type': 'text'},
{'id': 'author', 'type': 'text'},
{'id': 'published'}],
'records': [{u'b\xfck': 'annakarenina',
'author': 'tolstoy',
'published': '2005-03-01',
'nested': ['b', {'moo': 'moo'}]},
{u'b\xfck': 'warandpeace',
'author': 'tolstoy',
'nested': {'a': 'b'}}]
}
postparams = '%s=1' % json.dumps(cls.data)
auth = {'Authorization': str(cls.sysadmin_user.apikey)}
res = cls.app.post('/api/action/datastore_create', params=postparams,
extra_environ=auth)
res_dict = json.loads(res.body)
assert res_dict['success'] is True
# Make an organization, because private datasets must belong to one.
cls.organization = tests.call_action_api(
cls.app, 'organization_create',
name='test_org',
apikey=cls.sysadmin_user.apikey)
cls.expected_records = [{u'_full_text': [u"'annakarenina'", u"'b'",
u"'moo'", u"'tolstoy'",
u"'2005'"],
u'_id': 1,
u'author': u'tolstoy',
u'b\xfck': u'annakarenina',
u'nested': [u'b', {u'moo': u'moo'}],
u'published': u'2005-03-01T00:00:00'},
{u'_full_text': [u"'tolstoy'", u"'warandpeac'",
u"'b'"],
u'_id': 2,
u'author': u'tolstoy',
u'b\xfck': u'warandpeace',
u'nested': {u'a': u'b'},
u'published': None}]
cls.expected_join_results = [{u'first': 1, u'second': 1}, {u'first': 1, u'second': 2}]
engine = db.get_write_engine()
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
def test_get_function_names(self):
engine = db.get_write_engine()
session = orm.scoped_session(orm.sessionmaker(bind=engine))
create_tables = [
u"CREATE TABLE test_a (id int, period date, subject_id text, result decimal)",
u"CREATE TABLE test_b (name text, subject_id text)",
]
for create_table_sql in create_tables:
session.execute(create_table_sql)
test_cases = [
(u"SELECT max(id) from test_a", ["max"]),
(u"SELECT count(distinct(id)) FROM test_a", ["count", "distinct"]),
(u"SELECT trunc(avg(result),2) FROM test_a", ["trunc", "avg"]),
(u"SELECT query_to_xml('SELECT max(id) FROM test_a', true, true , '')",
["query_to_xml"]),
(u"select $$'$$, query_to_xml($X$SELECT table_name FROM information_schema.tables$X$,true,true,$X$$X$), $$'$$",
["query_to_xml"])
]
context = {"connection": session.connection()}
for case in test_cases:
assert sorted(
datastore_helpers.get_table_and_function_names_from_sql(
context, case[0])[1]) == sorted(case[1])
def setup_class(cls):
if not tests.is_datastore_supported():
raise nose.SkipTest("Datastore not supported")
plugin = p.load('datastore')
if plugin.legacy_mode:
# make sure we undo adding the plugin
p.unload('datastore')
raise nose.SkipTest("SQL tests are not supported in legacy mode")
ctd.CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.normal_user = model.User.get('annafan')
cls.dataset = model.Package.get('annakarenina')
resource = cls.dataset.resources[0]
cls.data = {
'resource_id': resource.id,
'force': True,
'aliases': 'books4',
'fields': [{'id': u'b\xfck', 'type': 'text'},
{'id': 'author', 'type': 'text'},
{'id': 'published'}],
'records': [{u'b\xfck': 'annakarenina',
'author': 'tolstoy',
'published': '2005-03-01',
'nested': ['b', {'moo': 'moo'}]},
{u'b\xfck': 'warandpeace',
'author': 'tolstoy',
'nested': {'a': 'b'}}]
}
postparams = '%s=1' % json.dumps(cls.data)
auth = {'Authorization': str(cls.sysadmin_user.apikey)}
res = cls.app.post('/api/action/datastore_create', params=postparams,
extra_environ=auth)
res_dict = json.loads(res.body)
assert res_dict['success'] is True
# Make an organization, because private datasets must belong to one.
cls.organization = tests.call_action_api(
cls.app, 'organization_create',
name='test_org',
apikey=cls.sysadmin_user.apikey)
cls.expected_records = [{u'_full_text': [u"'annakarenina'", u"'b'",
u"'moo'", u"'tolstoy'",
u"'2005'"],
u'_id': 1,
u'author': u'tolstoy',
u'b\xfck': u'annakarenina',
u'nested': [u'b', {u'moo': u'moo'}],
u'published': u'2005-03-01T00:00:00'},
{u'_full_text': [u"'tolstoy'", u"'warandpeac'",
u"'b'"],
u'_id': 2,
u'author': u'tolstoy',
u'b\xfck': u'warandpeace',
u'nested': {u'a': u'b'},
u'published': None}]
cls.expected_join_results = [{u'first': 1, u'second': 1}, {u'first': 1, u'second': 2}]
engine = db.get_write_engine()
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
def setup_class(cls):
if not p.plugin_loaded(u"datastore"):
p.load(u"datastore")
reset_db()
search.clear_all()
engine = db.get_write_engine()
rebuild_all_dbs(orm.scoped_session(orm.sessionmaker(bind=engine)))
def setup_class(cls):
if not tests.is_datastore_supported():
raise nose.SkipTest("Datastore not supported")
cls.app = helpers._get_test_app()
p.load('datastore')
ctd.CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.normal_user = model.User.get('annafan')
resource = model.Package.get('annakarenina').resources[0]
cls.data = {
'resource_id': resource.id,
'aliases': u'b\xfck2',
'fields': [{'id': 'book', 'type': 'text'},
{'id': 'author', 'type': 'text'},
{'id': 'rating with %', 'type': 'text'}],
'records': [{'book': 'annakarenina', 'author': 'tolstoy',
'rating with %': '90%'},
{'book': 'warandpeace', 'author': 'tolstoy',
'rating with %': '42%'}]
}
engine = db.get_write_engine()
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
set_url_type(
model.Package.get('annakarenina').resources, cls.sysadmin_user)
def setup_class(cls):
test_helpers.reset_db()
cls.app = test_helpers._get_test_app()
if not tests.is_datastore_supported():
raise nose.SkipTest("Datastore not supported")
p.load('datastore')
ctd.CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.normal_user = model.User.get('annafan')
resource = model.Package.get('annakarenina').resources[0]
cls.data = {
'resource_id':
resource.id,
'force':
True,
'aliases':
'books',
'fields': [{
'id': u'b\xfck',
'type': 'text'
}, {
'id': 'author',
'type': 'text'
}, {
'id': 'published'
}, {
'id': u'characters',
u'type': u'_text'
}, {
'id': 'random_letters',
'type': 'text[]'
}],
'records': [{
u'b\xfck': 'annakarenina',
'author': 'tolstoy',
'published': '2005-03-01',
'nested': ['b', {
'moo': 'moo'
}],
u'characters': [u'Princess Anna', u'Sergius'],
'random_letters': ['a', 'e', 'x']
}, {
u'b\xfck': 'warandpeace',
'author': 'tolstoy',
'nested': {
'a': 'b'
},
'random_letters': []
}]
}
postparams = '%s=1' % json.dumps(cls.data)
auth = {'Authorization': str(cls.sysadmin_user.apikey)}
res = cls.app.post('/api/action/datastore_create',
params=postparams,
extra_environ=auth)
res_dict = json.loads(res.body)
assert res_dict['success'] is True
engine = db.get_write_engine()
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
def setup_class(cls):
if not tests.is_datastore_supported():
raise nose.SkipTest("Datastore not supported")
cls.app = helpers._get_test_app()
p.load('datastore')
ctd.CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.normal_user = model.User.get('annafan')
cls.dataset = model.Package.get('annakarenina')
cls.resource = cls.dataset.resources[0]
cls.data = {
'resource_id': cls.resource.id,
'force': True,
'aliases': 'books3',
'fields': [{'id': u'b\xfck', 'type': 'text'},
{'id': 'author', 'type': 'text'},
{'id': 'published'},
{'id': u'characters', u'type': u'_text'},
{'id': 'rating with %'}],
'records': [{u'b\xfck': 'annakarenina', 'author': 'tolstoy',
'published': '2005-03-01', 'nested': ['b', {'moo': 'moo'}],
u'characters': [u'Princess Anna', u'Sergius'],
'rating with %': '60%'},
{u'b\xfck': 'warandpeace', 'author': 'tolstoy',
'nested': {'a': 'b'}, 'rating with %': '99%'}
]
}
postparams = '%s=1' % json.dumps(cls.data)
auth = {'Authorization': str(cls.sysadmin_user.apikey)}
res = cls.app.post('/api/action/datastore_create', params=postparams,
extra_environ=auth)
res_dict = json.loads(res.body)
assert res_dict['success'] is True
# Make an organization, because private datasets must belong to one.
cls.organization = tests.call_action_api(
cls.app, 'organization_create',
name='test_org',
apikey=cls.sysadmin_user.apikey)
cls.expected_records = [{u'published': u'2005-03-01T00:00:00',
u'_id': 1,
u'nested': [u'b', {u'moo': u'moo'}],
u'b\xfck': u'annakarenina',
u'author': u'tolstoy',
u'characters': [u'Princess Anna', u'Sergius'],
u'rating with %': u'60%'},
{u'published': None,
u'_id': 2,
u'nested': {u'a': u'b'},
u'b\xfck': u'warandpeace',
u'author': u'tolstoy',
u'characters': None,
u'rating with %': u'99%'}]
engine = db.get_write_engine()
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
def setup_class(cls):
if not tests.is_datastore_supported():
raise nose.SkipTest("Datastore not supported")
cls.app = helpers._get_test_app()
p.load('datastore')
ctd.CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.normal_user = model.User.get('annafan')
cls.dataset = model.Package.get('annakarenina')
cls.resource = cls.dataset.resources[0]
cls.data = {
'resource_id': cls.resource.id,
'force': True,
'aliases': 'books3',
'fields': [{'id': u'b\xfck', 'type': 'text'},
{'id': 'author', 'type': 'text'},
{'id': 'published'},
{'id': u'characters', u'type': u'_text'},
{'id': 'rating with %'}],
'records': [{u'b\xfck': 'annakarenina', 'author': 'tolstoy',
'published': '2005-03-01', 'nested': ['b', {'moo': 'moo'}],
u'characters': [u'Princess Anna', u'Sergius'],
'rating with %': '60%'},
{u'b\xfck': 'warandpeace', 'author': 'tolstoy',
'nested': {'a': 'b'}, 'rating with %': '99%'}
]
}
postparams = '%s=1' % json.dumps(cls.data)
auth = {'Authorization': str(cls.sysadmin_user.apikey)}
res = cls.app.post('/api/action/datastore_create', params=postparams,
extra_environ=auth)
res_dict = json.loads(res.body)
assert res_dict['success'] is True
# Make an organization, because private datasets must belong to one.
cls.organization = tests.call_action_api(
cls.app, 'organization_create',
name='test_org',
apikey=cls.sysadmin_user.apikey)
cls.expected_records = [{u'published': u'2005-03-01T00:00:00',
u'_id': 1,
u'nested': [u'b', {u'moo': u'moo'}],
u'b\xfck': u'annakarenina',
u'author': u'tolstoy',
u'characters': [u'Princess Anna', u'Sergius'],
u'rating with %': u'60%'},
{u'published': None,
u'_id': 2,
u'nested': {u'a': u'b'},
u'b\xfck': u'warandpeace',
u'author': u'tolstoy',
u'characters': None,
u'rating with %': u'99%'}]
engine = db.get_write_engine()
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
def setup_class(self, clean_db, test_request_context):
resource = factories.Resource(url_type="datastore")
self.dataset = factories.Dataset(resources=[resource])
with test_request_context():
self.sysadmin_user = factories.User(name="testsysadmin",
sysadmin=True)
self.normal_user = factories.User(name="annafan")
engine = db.get_write_engine()
self.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
def setup_class(cls):
cls.app = _get_test_app()
super(TestDatastoreCreate, cls).setup_class()
ctd.CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.normal_user = model.User.get('annafan')
engine = db.get_write_engine()
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
set_url_type(
model.Package.get('annakarenina').resources, cls.sysadmin_user)
def setup_class(cls):
cls.app = _get_test_app()
super(TestDatastoreCreate, cls).setup_class()
ctd.CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.normal_user = model.User.get('annafan')
engine = db.get_write_engine()
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
set_url_type(
model.Package.get('annakarenina').resources, cls.sysadmin_user)
def create_test_data(self, clean_datastore, test_request_context):
ctd.CreateTestData.create()
self.sysadmin_user = model.User.get("testsysadmin")
self.normal_user = model.User.get("annafan")
engine = db.get_write_engine()
self.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
with test_request_context():
set_url_type(
model.Package.get("annakarenina").resources, self.sysadmin_user
)
def initial_data(self, clean_db, clean_index):
if not tests.is_datastore_supported():
pytest.skip("Datastore not supported")
ctd.CreateTestData.create()
self.sysadmin_user = model.User.get("testsysadmin")
self.normal_user = model.User.get("annafan")
engine = db.get_write_engine()
self.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
set_url_type(
model.Package.get("annakarenina").resources, self.sysadmin_user)
def setup_class(self, clean_db):
if not tests.is_datastore_supported():
pytest.skip("Datastore not supported")
resource = factories.Resource(url_type="datastore")
self.dataset = factories.Dataset(resources=[resource])
self.sysadmin_user = factories.User(name="testsysadmin", sysadmin=True)
self.normal_user = factories.User(name="annafan")
engine = db.get_write_engine()
self.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
def initial_data(self, clean_db, clean_index, test_request_context, app):
ctd.CreateTestData.create()
self.sysadmin_user = model.User.get("testsysadmin")
self.normal_user = model.User.get("annafan")
engine = db.get_write_engine()
self.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
with test_request_context():
set_url_type(
model.Package.get("annakarenina").resources, self.sysadmin_user
)
url = url_for('sitemap.view')
self.cont = app.get(url)
def initial_data(self, clean_db, clean_index, test_request_context):
ctd.CreateTestData.create()
self.sysadmin_user = factories.Sysadmin()
self.sysadmin_token = factories.APIToken(user=self.sysadmin_user)
self.normal_user = factories.User()
self.normal_user_token = factories.APIToken(user=self.normal_user)
engine = db.get_write_engine()
self.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
with test_request_context():
set_url_type(
model.Package.get("annakarenina").resources, self.sysadmin_user
)
def setup_class(cls):
cls.app = helpers._get_test_app()
if not tests.is_datastore_supported():
raise nose.SkipTest("Datastore not supported")
p.load('datastore')
p.load('datapusher')
ctd.CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.normal_user = model.User.get('annafan')
engine = db.get_write_engine()
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
set_url_type(
model.Package.get('annakarenina').resources, cls.sysadmin_user)
def setup_class(cls):
cls.app = helpers._get_test_app()
if not tests.is_datastore_supported():
raise nose.SkipTest("Datastore not supported")
p.load('datastore')
p.load('datapusher')
ctd.CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.normal_user = model.User.get('annafan')
engine = db.get_write_engine()
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
set_url_type(
model.Package.get('annakarenina').resources, cls.sysadmin_user)
def initial_data(self, clean_datastore, app, test_request_context):
self.app = app
ctd.CreateTestData.create()
self.sysadmin_user = factories.Sysadmin()
self.sysadmin_token = factories.APIToken(user=self.sysadmin_user["id"])
self.sysadmin_token = self.sysadmin_token["token"]
self.normal_user = factories.User()
self.normal_user_token = factories.APIToken(
user=self.normal_user["id"])
self.normal_user_token = self.normal_user_token["token"]
resource = model.Package.get("annakarenina").resources[0]
self.data = {
"resource_id":
resource.id,
"aliases":
u"b\xfck2",
"fields": [
{
"id": "book",
"type": "text"
},
{
"id": "author",
"type": "text"
},
{
"id": "rating with %",
"type": "text"
},
],
"records": [
{
"book": "annakarenina",
"author": "tolstoy",
"rating with %": "90%",
},
{
"book": "warandpeace",
"author": "tolstoy",
"rating with %": "42%",
},
],
}
engine = db.get_write_engine()
self.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
with test_request_context():
set_url_type(
model.Package.get("annakarenina").resources,
self.sysadmin_user)
def setup_class(cls):
wsgiapp = middleware.make_app(config['global_conf'], **config)
cls.app = paste.fixture.TestApp(wsgiapp)
if not tests.is_datastore_supported():
raise nose.SkipTest("Datastore not supported")
p.load('datastore')
p.load('datapusher')
ctd.CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.normal_user = model.User.get('annafan')
engine = db.get_write_engine()
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
set_url_type(
model.Package.get('annakarenina').resources, cls.sysadmin_user)
def querystore_resolve(self, pid, records_format='objects'):
qs = QueryStore()
query = qs.retrieve_query(pid)
if query is None:
return None
if resource_exists(query.resource_id):
connection = get_write_engine().connect()
rs = postgres_querystore_resolve(query)
#column names as a list
column_names = rs.keys()
#to delete standard sys_period column for versioning
#(is not necessary for the user, it has a technical meaning)
del column_names[-1]
search_result = refine_results(rs, column_names)
result_dictionary = {
'column_names': sorted(column_names),
'result_set': search_result,
'query': query,
'resource_id': query.resource_id
}
context = {'connection': connection}
fields_types = _get_fields_types(context['connection'],
query.resource_id)
result_dictionary['fields'] = sorted(
_result_fields(
fields_types,
_get_field_info(context['connection'], query.resource_id),
get_list(result_dictionary.get('fields'))))
# do not show sys_period column because, it has only
# a technical meaning
result_dictionary['fields'] = exclude_sys_period(
result_dictionary['fields'])
if records_format == 'objects':
result_dictionary['result_set'] = list(
result_dictionary['result_set'])
elif records_format == 'csv':
result_dictionary['result_set'] = convert_to_csv(
result_dictionary['result_set'])
return result_dictionary
else:
return {}
def setup_class(cls):
engine = db.get_write_engine()
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
datastore_test_helpers.clear_db(cls.Session)
create_tables = [
u'CREATE TABLE test_a (id_a text)',
u'CREATE TABLE test_b (id_b text)',
u'CREATE TABLE "TEST_C" (id_c text)',
u'CREATE TABLE test_d ("α/α" integer)',
]
for create_table_sql in create_tables:
cls.Session.execute(create_table_sql)
def setup_class(cls):
cls.app = _get_test_app()
if not tests.is_datastore_supported():
raise nose.SkipTest("Datastore not supported")
p.load('datastore')
p.load('datapusher')
p.load('test_datapusher_plugin')
resource = factories.Resource(url_type='datastore')
cls.dataset = factories.Dataset(resources=[resource])
cls.sysadmin_user = factories.User(name='testsysadmin', sysadmin=True)
cls.normal_user = factories.User(name='annafan')
engine = db.get_write_engine()
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
def setup_class(cls):
engine = db.get_write_engine()
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
datastore_test_helpers.clear_db(cls.Session)
create_tables = [
u'CREATE TABLE test_a (id_a text)',
u'CREATE TABLE test_b (id_b text)',
u'CREATE TABLE "TEST_C" (id_c text)',
u'CREATE TABLE test_d ("α/α" integer)',
]
for create_table_sql in create_tables:
cls.Session.execute(create_table_sql)
def setup_class(cls):
cls.app = _get_test_app()
if not tests.is_datastore_supported():
raise nose.SkipTest("Datastore not supported")
p.load('datastore')
p.load('datapusher')
p.load('test_datapusher_plugin')
resource = factories.Resource(url_type='datastore')
cls.dataset = factories.Dataset(resources=[resource])
cls.sysadmin_user = factories.User(name='testsysadmin', sysadmin=True)
cls.normal_user = factories.User(name='annafan')
engine = db.get_write_engine()
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
def setup_class(cls):
wsgiapp = middleware.make_app(config['global_conf'], **config)
cls.app = paste.fixture.TestApp(wsgiapp)
if not tests.is_datastore_supported():
raise nose.SkipTest("Datastore not supported")
p.load('datastore')
p.load('datapusher')
p.load('test_datapusher_plugin')
resource = factories.Resource(url_type='datastore')
cls.dataset = factories.Dataset(resources=[resource])
cls.sysadmin_user = factories.User(name='testsysadmin', sysadmin=True)
cls.normal_user = factories.User(name='annafan')
engine = db.get_write_engine()
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
def test_get_table_names(self):
engine = db.get_write_engine()
session = orm.scoped_session(orm.sessionmaker(bind=engine))
create_tables = [
u"CREATE TABLE test_a (id_a text)",
u"CREATE TABLE test_b (id_b text)",
u'CREATE TABLE "TEST_C" (id_c text)',
u'CREATE TABLE test_d ("α/α" integer)',
]
for create_table_sql in create_tables:
session.execute(create_table_sql)
test_cases = [
(u"SELECT * FROM test_a", ["test_a"]),
(u"SELECT * FROM public.test_a", ["test_a"]),
(u'SELECT * FROM "TEST_C"', ["TEST_C"]),
(u'SELECT * FROM public."TEST_C"', ["TEST_C"]),
(u"SELECT * FROM pg_catalog.pg_database", ["pg_database"]),
(u"SELECT rolpassword FROM pg_roles", ["pg_authid"]),
(
u"""SELECT p.rolpassword
FROM pg_roles p
JOIN test_b b
ON p.rolpassword = b.id_b""",
["pg_authid", "test_b"],
),
(
u"""SELECT id_a, id_b, id_c
FROM (
SELECT *
FROM (
SELECT *
FROM "TEST_C") AS c,
test_b) AS b,
test_a AS a""",
["test_a", "test_b", "TEST_C"],
),
(u"INSERT INTO test_a VALUES ('a')", ["test_a"]),
(u'SELECT "α/α" FROM test_d', ["test_d"]),
(u'SELECT "α/α" FROM test_d WHERE "α/α" > 1000', ["test_d"]),
]
context = {"connection": session.connection()}
for case in test_cases:
assert sorted(
datastore_helpers.get_table_names_from_sql(
context, case[0])) == sorted(case[1])
def setup_class(cls):
if not config.get('ckan.datastore.read_url'):
raise nose.SkipTest('Datastore runs on legacy mode, skipping...')
engine = db.get_write_engine()
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
datastore_test_helpers.clear_db(cls.Session)
create_tables = [
u'CREATE TABLE test_a (id_a text)',
u'CREATE TABLE test_b (id_b text)',
u'CREATE TABLE "TEST_C" (id_c text)',
u'CREATE TABLE test_d ("α/α" integer)',
]
for create_table_sql in create_tables:
cls.Session.execute(create_table_sql)
def initial_data(self, clean_datastore, app):
self.app = app
ctd.CreateTestData.create()
self.sysadmin_user = model.User.get("testsysadmin")
self.normal_user = model.User.get("annafan")
resource = model.Package.get("annakarenina").resources[0]
self.data = {
"resource_id":
resource.id,
"aliases":
u"b\xfck2",
"fields": [
{
"id": "book",
"type": "text"
},
{
"id": "author",
"type": "text"
},
{
"id": "rating with %",
"type": "text"
},
],
"records": [
{
"book": "annakarenina",
"author": "tolstoy",
"rating with %": "90%",
},
{
"book": "warandpeace",
"author": "tolstoy",
"rating with %": "42%",
},
],
}
engine = db.get_write_engine()
self.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
set_url_type(
model.Package.get("annakarenina").resources, self.sysadmin_user)
def setup_class(cls):
cls.app = helpers._get_test_app()
if not tests.is_datastore_supported():
raise nose.SkipTest("Datastore not supported")
p.load('datastore')
ctd.CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.normal_user = model.User.get('annafan')
resource = model.Package.get('annakarenina').resources[0]
cls.data = {
'resource_id':
resource.id,
'aliases':
u'b\xfck2',
'fields': [{
'id': 'book',
'type': 'text'
}, {
'id': 'author',
'type': 'text'
}, {
'id': 'rating with %',
'type': 'text'
}],
'records': [{
'book': 'annakarenina',
'author': 'tolstoy',
'rating with %': '90%'
}, {
'book': 'warandpeace',
'author': 'tolstoy',
'rating with %': '42%'
}]
}
engine = db.get_write_engine()
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
with cls.app.flask_app.test_request_context():
set_url_type(
model.Package.get('annakarenina').resources, cls.sysadmin_user)
def test_get_function_names_custom_function(self):
engine = db.get_write_engine()
session = orm.scoped_session(orm.sessionmaker(bind=engine))
create_tables = [
u"""CREATE FUNCTION add(integer, integer) RETURNS integer
AS 'select $1 + $2;'
LANGUAGE SQL
IMMUTABLE
RETURNS NULL ON NULL INPUT;
"""
]
for create_table_sql in create_tables:
session.execute(create_table_sql)
context = {"connection": session.connection()}
sql = "SELECT add(1, 2);"
assert datastore_helpers.get_table_and_function_names_from_sql(
context, sql)[1] == ["add"]
def setup_class(cls):
if not tests.is_datastore_supported():
raise nose.SkipTest("Datastore not supported")
cls.app = helpers._get_test_app()
p.load('datastore')
ctd.CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.normal_user = model.User.get('annafan')
set_url_type(
model.Package.get('annakarenina').resources, cls.sysadmin_user)
resource = model.Package.get('annakarenina').resources[0]
hhguide = u"hitchhiker's guide to the galaxy"
cls.data = {
'resource_id': resource.id,
'fields': [{'id': u'b\xfck', 'type': 'text'},
{'id': 'author', 'type': 'text'},
{'id': 'nested', 'type': 'json'},
{'id': 'characters', 'type': 'text[]'},
{'id': 'published'}],
'primary_key': u'b\xfck',
'records': [{u'b\xfck': 'annakarenina', 'author': 'tolstoy',
'published': '2005-03-01', 'nested': ['b', {'moo': 'moo'}]},
{u'b\xfck': 'warandpeace', 'author': 'tolstoy',
'nested': {'a':'b'}},
{'author': 'adams',
'characters': ['Arthur Dent', 'Marvin'],
'nested': {'foo': 'bar'},
u'b\xfck': hhguide}
]
}
postparams = '%s=1' % json.dumps(cls.data)
auth = {'Authorization': str(cls.sysadmin_user.apikey)}
res = cls.app.post('/api/action/datastore_create', params=postparams,
extra_environ=auth)
res_dict = json.loads(res.body)
assert res_dict['success'] is True
engine = db.get_write_engine()
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
def setup_class(cls):
if not tests.is_datastore_supported():
raise nose.SkipTest("Datastore not supported")
p.load('datastore')
ctd.CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.normal_user = model.User.get('annafan')
set_url_type(
model.Package.get('annakarenina').resources, cls.sysadmin_user)
resource = model.Package.get('annakarenina').resources[0]
hhguide = u"hitchhiker's guide to the galaxy"
cls.data = {
'resource_id': resource.id,
'fields': [{'id': u'b\xfck', 'type': 'text'},
{'id': 'author', 'type': 'text'},
{'id': 'nested', 'type': 'json'},
{'id': 'characters', 'type': 'text[]'},
{'id': 'published'}],
'primary_key': u'b\xfck',
'records': [{u'b\xfck': 'annakarenina', 'author': 'tolstoy',
'published': '2005-03-01', 'nested': ['b', {'moo': 'moo'}]},
{u'b\xfck': 'warandpeace', 'author': 'tolstoy',
'nested': {'a':'b'}},
{'author': 'adams',
'characters': ['Arthur Dent', 'Marvin'],
'nested': {'foo': 'bar'},
u'b\xfck': hhguide}
]
}
postparams = '%s=1' % json.dumps(cls.data)
auth = {'Authorization': str(cls.sysadmin_user.apikey)}
res = cls.app.post('/api/action/datastore_create', params=postparams,
extra_environ=auth)
res_dict = json.loads(res.body)
assert res_dict['success'] is True
engine = db.get_write_engine()
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
def test_get_function_names_crosstab(self):
"""
Crosstab functions need to be enabled in the database by executing the following using
a super user:
CREATE extension tablefunc;
"""
engine = db.get_write_engine()
session = orm.scoped_session(orm.sessionmaker(bind=engine))
create_tables = [
u"CREATE TABLE test_a (id int, period date, subject_id text, result decimal)",
u"CREATE TABLE test_b (name text, subject_id text)",
]
for create_table_sql in create_tables:
session.execute(create_table_sql)
test_cases = [
(u"""SELECT *
FROM crosstab(
'SELECT extract(month from period)::text, test_b.name, trunc(avg(result),2)
FROM test_a, test_b
WHERE test_a.subject_id = test_b.subject_id')
AS final_result(month text, subject_1 numeric,subject_2 numeric);""",
['crosstab', 'final_result', 'extract', 'trunc', 'avg']),
]
context = {"connection": session.connection()}
try:
for case in test_cases:
assert sorted(
datastore_helpers.get_table_and_function_names_from_sql(
context, case[0])[1]) == sorted(case[1])
except ProgrammingError as e:
if bool(re.search("function crosstab(.*) does not exist", str(e))):
pytest.skip(
"crosstab functions not enabled in DataStore database")
def setup_class(cls):
cls.app = helpers._get_test_app()
super(TestDatastoreDelete, cls).setup_class()
ctd.CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.normal_user = model.User.get('annafan')
resource = model.Package.get('annakarenina').resources[0]
cls.data = {
'resource_id':
resource.id,
'aliases':
u'b\xfck2',
'fields': [{
'id': 'book',
'type': 'text'
}, {
'id': 'author',
'type': 'text'
}, {
'id': 'rating with %',
'type': 'text'
}],
'records': [{
'book': 'annakarenina',
'author': 'tolstoy',
'rating with %': '90%'
}, {
'book': 'warandpeace',
'author': 'tolstoy',
'rating with %': '42%'
}]
}
engine = db.get_write_engine()
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
set_url_type(
model.Package.get('annakarenina').resources, cls.sysadmin_user)
def postgres_querystore_resolve(query):
engine = get_write_engine()
connection = engine.connect()
if query:
if 'WHERE' in query.query:
where = u'''WHERE (lower(sys_period) <='{exec_timestamp}') AND (('{exec_timestamp}' < upper(sys_period)) OR upper(sys_period) IS NULL) AND'''.format(
exec_timestamp=query.exec_timestamp)
history_query = (query.query.replace(query.resource_id, query.resource_id + '_history')).replace('WHERE',
where)
select = u'''{query}
UNION {history_query} ORDER BY _id'''.format(query=query.query.replace('WHERE', where),
history_query=history_query)
else:
where = u'''WHERE (lower(sys_period) <='{exec_timestamp}') AND (('{exec_timestamp}' < upper(sys_period)) OR upper(sys_period) IS NULL)'''.format(
exec_timestamp=query.exec_timestamp)
history_query = (query.query.replace(query.resource_id, query.resource_id + '_history')) + ' ' + where
select = u'''{query} UNION {history_query} ORDER BY _id'''.format(query=query.query + ' ' + where,
history_query=history_query)
result = connection.execute(select)
return result
else:
return None
def _execute_sql(self, sql, *args):
engine = db.get_write_engine()
session = orm.scoped_session(orm.sessionmaker(bind=engine))
return session.connection().execute(sql, *args)
def setup_class(cls):
engine = db.get_write_engine()
rebuild_all_dbs(orm.scoped_session(orm.sessionmaker(bind=engine)))
super(DatastoreFunctionalTestBase, cls).setup_class()
def execute_sql(sql, *args):
engine = db.get_write_engine()
session = orm.scoped_session(orm.sessionmaker(bind=engine))
return session.connection().execute(sql, *args)
def setup_class(cls):
engine = db.get_write_engine()
rebuild_all_dbs(orm.scoped_session(orm.sessionmaker(bind=engine)))
super(DatastoreFunctionalTestBase, cls).setup_class()
def setup_class(cls):
p.load(u'datastore')
reset_db()
search.clear_all()
engine = db.get_write_engine()
rebuild_all_dbs(orm.scoped_session(orm.sessionmaker(bind=engine)))
def setup_class(cls):
wsgiapp = middleware.make_app(config['global_conf'], **config)
cls.app = paste.fixture.TestApp(wsgiapp)
if not tests.is_datastore_supported():
raise nose.SkipTest("Datastore not supported")
p.load('datastore')
ctd.CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.normal_user = model.User.get('annafan')
resource = model.Package.get('annakarenina').resources[0]
cls.data = {
'resource_id': resource.id,
'force': True,
'aliases': 'books',
'fields': [
{
'id': u'b\xfck',
'type': 'text'
},
{
'id': 'author',
'type': 'text'
},
{
'id': 'published'
},
{
'id': u'characters',
u'type': u'_text'
},
{
'id': 'random_letters',
'type': 'text[]'
}
],
'records': [
{
u'b\xfck': 'annakarenina',
'author': 'tolstoy',
'published': '2005-03-01',
'nested': [
'b',
{'moo': 'moo'}
],
u'characters': [
u'Princess Anna',
u'Sergius'
],
'random_letters': [
'a', 'e', 'x'
]
},
{
u'b\xfck': 'warandpeace',
'author': 'tolstoy',
'nested': {'a': 'b'},
'random_letters': [
]
}
]
}
postparams = '%s=1' % json.dumps(cls.data)
auth = {'Authorization': str(cls.sysadmin_user.apikey)}
res = cls.app.post('/api/action/datastore_create', params=postparams,
extra_environ=auth)
res_dict = json.loads(res.body)
assert res_dict['success'] is True
engine = db.get_write_engine()
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
