def configure(self, config):
self.config = config
# check for ckan.datastore.write_url and ckan.datastore.read_url
if 'ckan.datastore.write_url' not in config:
error_msg = 'ckan.datastore.write_url not found in config'
raise DatastoreException(error_msg)
# Legacy mode means that we have no read url. Consequently sql search
# is not available and permissions do not have to be changed. In legacy
# mode, the datastore runs on PG prior to 9.0 (for example 8.4).
self.legacy_mode = _is_legacy_mode(self.config)
# Check whether users have disabled datastore_search_sql
self.enable_sql_search = p.toolkit.asbool(
self.config.get('ckan.datastore.sqlsearch.enabled', True))
datapusher_formats = config.get('datapusher.formats', '').split()
self.datapusher_formats = datapusher_formats or DEFAULT_FORMATS
# Check whether we are running one of the paster commands which means
# that we should ignore the following tests.
if sys.argv[0].split('/')[-1] == 'paster':
if 'datastore' in sys.argv[1:]:
log.warn(
'Omitting permission checks because you are running'
'paster commands.'
)
return
self.ckan_url = self.config['sqlalchemy.url']
self.write_url = self.config['ckan.datastore.write_url']
if self.legacy_mode:
self.read_url = self.write_url
log.warn('Legacy mode active. '
'The sql search will not be available.')
else:
self.read_url = self.config['ckan.datastore.read_url']
self.read_engine = db._get_engine(
{'connection_url': self.read_url})
if not model.engine_is_pg(self.read_engine):
log.warn('We detected that you do not use a PostgreSQL '
'database. The DataStore will NOT work and DataStore '
'tests will be skipped.')
return
if self._is_read_only_database():
log.warn('We detected that CKAN is running on a read '
'only database. Permission checks and the creation '
'of _table_metadata are skipped.')
else:
self._check_urls_and_permissions()
self._create_alias_table()
def configure(self, config):
self.config = config
# check for ckan.datastore.write_url and ckan.datastore.read_url
if (not 'ckan.datastore.write_url' in config):
error_msg = 'ckan.datastore.write_url not found in config'
raise DatastoreException(error_msg)
# Legacy mode means that we have no read url. Consequently sql search is not
# available and permissions do not have to be changed. In legacy mode, the
# datastore runs on PG prior to 9.0 (for example 8.4).
self.legacy_mode = _is_legacy_mode(self.config)
# Check whether users have disabled datastore_search_sql
self.enable_sql_search = p.toolkit.asbool(
self.config.get('ckan.datastore.sqlsearch.enabled', True))
datapusher_formats = config.get('datapusher.formats', '').split()
self.datapusher_formats = datapusher_formats or DEFAULT_FORMATS
# Check whether we are running one of the paster commands which means
# that we should ignore the following tests.
if sys.argv[0].split(
'/')[-1] == 'paster' and 'timeseries' in sys.argv[1:]:
log.warn('Omitting permission checks because you are '
'running paster commands.')
return
self.ckan_url = self.config['sqlalchemy.url']
self.write_url = self.config['ckan.datastore.write_url']
if self.legacy_mode:
self.read_url = self.write_url
log.warn('Legacy mode active. '
'The sql search will not be available.')
else:
self.read_url = self.config['ckan.datastore.read_url']
self.read_engine = db._get_engine({'connection_url': self.read_url})
if not model.engine_is_pg(self.read_engine):
log.warn('We detected that you do not use a PostgreSQL '
'database. The DataStore will NOT work and DataStore '
'tests will be skipped.')
return
if self._is_read_only_database():
log.warn('We detected that CKAN is running on a read '
'only database. Permission checks and the creation '
'of _table_metadata are skipped.')
else:
self._check_urls_and_permissions()
self._create_alias_table(
) # create another _table_metadata_ts for resource size
def configure(self, config):
self.config = config
# check for ckan.datastore.write_url and ckan.datastore.read_url
if not "ckan.datastore.write_url" in config:
error_msg = "ckan.datastore.write_url not found in config"
raise DatastoreException(error_msg)
# Legacy mode means that we have no read url. Consequently sql search is not
# available and permissions do not have to be changed. In legacy mode, the
# datastore runs on PG prior to 9.0 (for example 8.4).
self.legacy_mode = _is_legacy_mode(self.config)
datapusher_formats = config.get("datapusher.formats", "").split()
self.datapusher_formats = datapusher_formats or DEFAULT_FORMATS
# Check whether we are running one of the paster commands which means
# that we should ignore the following tests.
if sys.argv[0].split("/")[-1] == "paster" and "datastore" in sys.argv[1:]:
log.warn("Omitting permission checks because you are " "running paster commands.")
return
self.ckan_url = self.config["sqlalchemy.url"]
self.write_url = self.config["ckan.datastore.write_url"]
if self.legacy_mode:
self.read_url = self.write_url
log.warn("Legacy mode active. " "The sql search will not be available.")
else:
self.read_url = self.config["ckan.datastore.read_url"]
self.read_engine = db._get_engine({"connection_url": self.read_url})
if not model.engine_is_pg(self.read_engine):
log.warn(
"We detected that you do not use a PostgreSQL "
"database. The DataStore will NOT work and DataStore "
"tests will be skipped."
)
return
if self._is_read_only_database():
log.warn(
"We detected that CKAN is running on a read "
"only database. Permission checks and the creation "
"of _table_metadata are skipped."
)
else:
self._check_urls_and_permissions()
self._create_alias_table()
def is_datastore_supported():
# we assume that the datastore uses the same db engine that ckan uses
is_supported_db = model.engine_is_pg()
return is_supported_db
def is_datastore_supported():
is_supported_db = model.engine_is_pg()
return is_supported_db
def configure(self, config):
self.config = config
# check for ckan.datastore.write_url and ckan.datastore.read_url
if (not 'ckan.datastore.write_url' in config):
error_msg = 'ckan.datastore.write_url not found in config'
raise DatastoreException(error_msg)
# Legacy mode means that we have no read url. Consequently sql search is not
# available and permissions do not have to be changed. In legacy mode, the
# datastore runs on PG prior to 9.0 (for example 8.4).
self.legacy_mode = 'ckan.datastore.read_url' not in self.config
# Check whether we are running one of the paster commands which means
# that we should ignore the following tests.
if sys.argv[0].split('/')[-1] == 'paster' and 'datastore' in sys.argv[1:]:
log.warn('Omitting permission checks because you are '
'running paster commands.')
return
self.ckan_url = self.config['sqlalchemy.url']
self.write_url = self.config['ckan.datastore.write_url']
if self.legacy_mode:
self.read_url = self.write_url
log.warn('Legacy mode active. '
'The sql search will not be available.')
else:
self.read_url = self.config['ckan.datastore.read_url']
read_engine = db._get_engine(
{'connection_url': self.read_url})
if not model.engine_is_pg(read_engine):
log.warn('We detected that you do not use a PostgreSQL '
'database. The DataStore will NOT work and DataStore '
'tests will be skipped.')
return
if self._is_read_only_database():
log.warn('We detected that CKAN is running on a read '
'only database. Permission checks and the creation '
'of _table_metadata are skipped.')
else:
self._check_urls_and_permissions()
self._create_alias_table()
# update the resource_show action to have datastore_active property
if self.resource_show_action is None:
resource_show = p.toolkit.get_action('resource_show')
@logic.side_effect_free
def new_resource_show(context, data_dict):
new_data_dict = resource_show(context, data_dict)
try:
connection = read_engine.connect()
result = connection.execute(
'SELECT 1 FROM "_table_metadata" WHERE name = %s AND alias_of IS NULL',
new_data_dict['id']
).fetchone()
if result:
new_data_dict['datastore_active'] = True
else:
new_data_dict['datastore_active'] = False
finally:
connection.close()
return new_data_dict
self.resource_show_action = new_resource_show
def is_datastore_supported():
# we assume that the datastore uses the same db engine that ckan uses
is_supported_db = model.engine_is_pg()
return is_supported_db
def configure(self, config):
self.config = config
# check for ckan.datastore.write_url and ckan.datastore.read_url
if (not 'ckan.datastore.write_url' in config):
error_msg = 'ckan.datastore.write_url not found in config'
raise DatastoreException(error_msg)
# Legacy mode means that we have no read url. Consequently sql search is not
# available and permissions do not have to be changed. In legacy mode, the
# datastore runs on PG prior to 9.0 (for example 8.4).
self.legacy_mode = 'ckan.datastore.read_url' not in self.config
# Check whether we are running one of the paster commands which means
# that we should ignore the following tests.
import sys
if sys.argv[0].split('/')[-1] == 'paster' and 'datastore' in sys.argv[1:]:
log.warn('Omitting permission checks because you are '
'running paster commands.')
return
self.ckan_url = self.config['sqlalchemy.url']
self.write_url = self.config['ckan.datastore.write_url']
if self.legacy_mode:
self.read_url = self.write_url
else:
self.read_url = self.config['ckan.datastore.read_url']
if model.engine_is_pg():
if not self._is_read_only_database():
# Make sure that the right permissions are set
# so that no harmful queries can be made
if not ('debug' in config and config['debug']):
self._check_separate_db()
if self.legacy_mode:
log.warn('Legacy mode active. The sql search will not be available.')
else:
self._check_read_permissions()
self._create_alias_table()
else:
log.warn("We detected that CKAN is running on a read only database. "
"Permission checks and the creation of _table_metadata are skipped.")
else:
log.warn("We detected that you do not use a PostgreSQL database. "
"The DataStore will NOT work and datastore tests will be skipped.")
## Do light wrapping around action function to add datastore_active
## to resource dict. Not using IAction extension as this prevents other plugins
## from having a custom resource_read.
# Make sure actions are cached
resource_show = p.toolkit.get_action('resource_show')
def new_resource_show(context, data_dict):
engine = db._get_engine(
context,
{'connection_url': self.read_url}
)
new_data_dict = resource_show(context, data_dict)
try:
connection = engine.connect()
result = connection.execute(
'SELECT 1 FROM "_table_metadata" WHERE name = %s AND alias_of IS NULL',
new_data_dict['id']
).fetchone()
if result:
new_data_dict['datastore_active'] = True
else:
new_data_dict['datastore_active'] = False
finally:
connection.close()
return new_data_dict
## Make sure do not run many times if configure is called repeatedly
## as in tests.
if not hasattr(resource_show, '_datastore_wrapped'):
new_resource_show._datastore_wrapped = True
logic._actions['resource_show'] = new_resource_show
def configure(self, config):
self.config = config
# check for ckan.datastore.write_url and ckan.datastore.read_url
if (not 'ckan.datastore.write_url' in config):
error_msg = 'ckan.datastore.write_url not found in config'
raise DatastoreException(error_msg)
# Legacy mode means that we have no read url. Consequently sql search is not
# available and permissions do not have to be changed. In legacy mode, the
# datastore runs on PG prior to 9.0 (for example 8.4).
self.legacy_mode = 'ckan.datastore.read_url' not in self.config
datapusher_formats = config.get('datapusher.formats', '').split()
self.datapusher_formats = datapusher_formats or DEFAULT_FORMATS
# Check whether we are running one of the paster commands which means
# that we should ignore the following tests.
if sys.argv[0].split('/')[-1] == 'paster' and 'datastore' in sys.argv[1:]:
log.warn('Omitting permission checks because you are '
'running paster commands.')
return
self.ckan_url = self.config['sqlalchemy.url']
self.write_url = self.config['ckan.datastore.write_url']
if self.legacy_mode:
self.read_url = self.write_url
log.warn('Legacy mode active. '
'The sql search will not be available.')
else:
self.read_url = self.config['ckan.datastore.read_url']
read_engine = db._get_engine(
{'connection_url': self.read_url})
if not model.engine_is_pg(read_engine):
log.warn('We detected that you do not use a PostgreSQL '
'database. The DataStore will NOT work and DataStore '
'tests will be skipped.')
return
if self._is_read_only_database():
log.warn('We detected that CKAN is running on a read '
'only database. Permission checks and the creation '
'of _table_metadata are skipped.')
else:
self._check_urls_and_permissions()
self._create_alias_table()
# update the resource_show action to have datastore_active property
if self.resource_show_action is None:
resource_show = p.toolkit.get_action('resource_show')
@logic.side_effect_free
def new_resource_show(context, data_dict):
new_data_dict = resource_show(context, data_dict)
try:
connection = read_engine.connect()
result = connection.execute(
'SELECT 1 FROM "_table_metadata" WHERE name = %s AND alias_of IS NULL',
new_data_dict['id']
).fetchone()
if result:
new_data_dict['datastore_active'] = True
else:
new_data_dict['datastore_active'] = False
finally:
connection.close()
return new_data_dict
self.resource_show_action = new_resource_show
def is_datastore_supported():
is_supported_db = model.engine_is_pg()
return is_supported_db
def configure(self, config):
self.config = config
# check for ckan.datastore.write_url and ckan.datastore.read_url
if not "ckan.datastore.write_url" in config:
error_msg = "ckan.datastore.write_url not found in config"
raise DatastoreException(error_msg)
# Legacy mode means that we have no read url. Consequently sql search is not
# available and permissions do not have to be changed. In legacy mode, the
# datastore runs on PG prior to 9.0 (for example 8.4).
self.legacy_mode = "ckan.datastore.read_url" not in self.config
# Check whether we are running one of the paster commands which means
# that we should ignore the following tests.
import sys
if sys.argv[0].split("/")[-1] == "paster" and "datastore" in sys.argv[1:]:
log.warn("Omitting permission checks because you are " "running paster commands.")
return
self.ckan_url = self.config["sqlalchemy.url"]
self.write_url = self.config["ckan.datastore.write_url"]
if self.legacy_mode:
self.read_url = self.write_url
log.warn("Legacy mode active. " "The sql search will not be available.")
else:
self.read_url = self.config["ckan.datastore.read_url"]
if not model.engine_is_pg():
log.warn(
"We detected that you do not use a PostgreSQL "
"database. The DataStore will NOT work and DataStore "
"tests will be skipped."
)
return
if self._is_read_only_database():
log.warn(
"We detected that CKAN is running on a read "
"only database. Permission checks and the creation "
"of _table_metadata are skipped."
)
else:
self._check_urls_and_permissions()
self._create_alias_table()
## Do light wrapping around action function to add datastore_active
## to resource dict. Not using IAction extension as this prevents
## other plugins from having a custom resource_show.
# Make sure actions are cached
resource_show = p.toolkit.get_action("resource_show")
@logic.side_effect_free
def new_resource_show(context, data_dict):
engine = db._get_engine(context, {"connection_url": self.read_url})
new_data_dict = resource_show(context, data_dict)
try:
connection = engine.connect()
result = connection.execute(
'SELECT 1 FROM "_table_metadata" WHERE name = %s AND alias_of IS NULL', new_data_dict["id"]
).fetchone()
if result:
new_data_dict["datastore_active"] = True
else:
new_data_dict["datastore_active"] = False
finally:
connection.close()
return new_data_dict
## Make sure do not run many times if configure is called repeatedly
## as in tests.
if not hasattr(resource_show, "_datastore_wrapped"):
new_resource_show._datastore_wrapped = True
logic._actions["resource_show"] = new_resource_show
context['schema'] = package_schema
if status == 'new':
# We need to explicitly provide a package ID, otherwise
# ckanext-spatial won't be be able to link the extent to the
# package.
if not package_dict.get('id'):
package_dict['id'] = unicode(uuid.uuid4())
package_schema['id'] = [unicode]
# Save reference to the package on the object
harvest_object.package_id = package_dict['id']
# Defer constraints and flush so the dataset can be indexed with
# the harvest object id (on the after_show hook from the harvester
# plugin)
if model.engine_is_pg():
model.Session.execute(
'SET CONSTRAINTS harvest_object_package_id_fkey DEFERRED')
model.Session.flush()
if source_config.get('private_datasets', False):
package_dict['private'] = True
log.debug('package_create: %r', package_dict)
try:
package_dict_created = tk.get_action('package_create')(
context.copy(), package_dict)
log.info('Created new package name=%s id=%s guid=%s',
package_dict.get('name'), package_dict_created['id'],
harvest_object.guid)
except tk.ValidationError, e:
