def configure(self, config): self.config = config # check for ckan.datastore.write_url and ckan.datastore.read_url if 'ckan.datastore.write_url' not in config: error_msg = 'ckan.datastore.write_url not found in config' raise DatastoreException(error_msg) # Legacy mode means that we have no read url. Consequently sql search # is not available and permissions do not have to be changed. In legacy # mode, the datastore runs on PG prior to 9.0 (for example 8.4). self.legacy_mode = _is_legacy_mode(self.config) # Check whether users have disabled datastore_search_sql self.enable_sql_search = p.toolkit.asbool( self.config.get('ckan.datastore.sqlsearch.enabled', True)) datapusher_formats = config.get('datapusher.formats', '').split() self.datapusher_formats = datapusher_formats or DEFAULT_FORMATS # Check whether we are running one of the paster commands which means # that we should ignore the following tests. if sys.argv[0].split('/')[-1] == 'paster': if 'datastore' in sys.argv[1:]: log.warn( 'Omitting permission checks because you are running' 'paster commands.' ) return self.ckan_url = self.config['sqlalchemy.url'] self.write_url = self.config['ckan.datastore.write_url'] if self.legacy_mode: self.read_url = self.write_url log.warn('Legacy mode active. ' 'The sql search will not be available.') else: self.read_url = self.config['ckan.datastore.read_url'] self.read_engine = db._get_engine( {'connection_url': self.read_url}) if not model.engine_is_pg(self.read_engine): log.warn('We detected that you do not use a PostgreSQL ' 'database. The DataStore will NOT work and DataStore ' 'tests will be skipped.') return if self._is_read_only_database(): log.warn('We detected that CKAN is running on a read ' 'only database. Permission checks and the creation ' 'of _table_metadata are skipped.') else: self._check_urls_and_permissions() self._create_alias_table()
def configure(self, config): self.config = config # check for ckan.datastore.write_url and ckan.datastore.read_url if (not 'ckan.datastore.write_url' in config): error_msg = 'ckan.datastore.write_url not found in config' raise DatastoreException(error_msg) # Legacy mode means that we have no read url. Consequently sql search is not # available and permissions do not have to be changed. In legacy mode, the # datastore runs on PG prior to 9.0 (for example 8.4). self.legacy_mode = _is_legacy_mode(self.config) # Check whether users have disabled datastore_search_sql self.enable_sql_search = p.toolkit.asbool( self.config.get('ckan.datastore.sqlsearch.enabled', True)) datapusher_formats = config.get('datapusher.formats', '').split() self.datapusher_formats = datapusher_formats or DEFAULT_FORMATS # Check whether we are running one of the paster commands which means # that we should ignore the following tests. if sys.argv[0].split( '/')[-1] == 'paster' and 'timeseries' in sys.argv[1:]: log.warn('Omitting permission checks because you are ' 'running paster commands.') return self.ckan_url = self.config['sqlalchemy.url'] self.write_url = self.config['ckan.datastore.write_url'] if self.legacy_mode: self.read_url = self.write_url log.warn('Legacy mode active. ' 'The sql search will not be available.') else: self.read_url = self.config['ckan.datastore.read_url'] self.read_engine = db._get_engine({'connection_url': self.read_url}) if not model.engine_is_pg(self.read_engine): log.warn('We detected that you do not use a PostgreSQL ' 'database. The DataStore will NOT work and DataStore ' 'tests will be skipped.') return if self._is_read_only_database(): log.warn('We detected that CKAN is running on a read ' 'only database. Permission checks and the creation ' 'of _table_metadata are skipped.') else: self._check_urls_and_permissions() self._create_alias_table( ) # create another _table_metadata_ts for resource size
def configure(self, config): self.config = config # check for ckan.datastore.write_url and ckan.datastore.read_url if not "ckan.datastore.write_url" in config: error_msg = "ckan.datastore.write_url not found in config" raise DatastoreException(error_msg) # Legacy mode means that we have no read url. Consequently sql search is not # available and permissions do not have to be changed. In legacy mode, the # datastore runs on PG prior to 9.0 (for example 8.4). self.legacy_mode = _is_legacy_mode(self.config) datapusher_formats = config.get("datapusher.formats", "").split() self.datapusher_formats = datapusher_formats or DEFAULT_FORMATS # Check whether we are running one of the paster commands which means # that we should ignore the following tests. if sys.argv[0].split("/")[-1] == "paster" and "datastore" in sys.argv[1:]: log.warn("Omitting permission checks because you are " "running paster commands.") return self.ckan_url = self.config["sqlalchemy.url"] self.write_url = self.config["ckan.datastore.write_url"] if self.legacy_mode: self.read_url = self.write_url log.warn("Legacy mode active. " "The sql search will not be available.") else: self.read_url = self.config["ckan.datastore.read_url"] self.read_engine = db._get_engine({"connection_url": self.read_url}) if not model.engine_is_pg(self.read_engine): log.warn( "We detected that you do not use a PostgreSQL " "database. The DataStore will NOT work and DataStore " "tests will be skipped." ) return if self._is_read_only_database(): log.warn( "We detected that CKAN is running on a read " "only database. Permission checks and the creation " "of _table_metadata are skipped." ) else: self._check_urls_and_permissions() self._create_alias_table()
def is_datastore_supported(): # we assume that the datastore uses the same db engine that ckan uses is_supported_db = model.engine_is_pg() return is_supported_db
def is_datastore_supported(): is_supported_db = model.engine_is_pg() return is_supported_db
def configure(self, config): self.config = config # check for ckan.datastore.write_url and ckan.datastore.read_url if (not 'ckan.datastore.write_url' in config): error_msg = 'ckan.datastore.write_url not found in config' raise DatastoreException(error_msg) # Legacy mode means that we have no read url. Consequently sql search is not # available and permissions do not have to be changed. In legacy mode, the # datastore runs on PG prior to 9.0 (for example 8.4). self.legacy_mode = 'ckan.datastore.read_url' not in self.config # Check whether we are running one of the paster commands which means # that we should ignore the following tests. if sys.argv[0].split('/')[-1] == 'paster' and 'datastore' in sys.argv[1:]: log.warn('Omitting permission checks because you are ' 'running paster commands.') return self.ckan_url = self.config['sqlalchemy.url'] self.write_url = self.config['ckan.datastore.write_url'] if self.legacy_mode: self.read_url = self.write_url log.warn('Legacy mode active. ' 'The sql search will not be available.') else: self.read_url = self.config['ckan.datastore.read_url'] read_engine = db._get_engine( {'connection_url': self.read_url}) if not model.engine_is_pg(read_engine): log.warn('We detected that you do not use a PostgreSQL ' 'database. The DataStore will NOT work and DataStore ' 'tests will be skipped.') return if self._is_read_only_database(): log.warn('We detected that CKAN is running on a read ' 'only database. Permission checks and the creation ' 'of _table_metadata are skipped.') else: self._check_urls_and_permissions() self._create_alias_table() # update the resource_show action to have datastore_active property if self.resource_show_action is None: resource_show = p.toolkit.get_action('resource_show') @logic.side_effect_free def new_resource_show(context, data_dict): new_data_dict = resource_show(context, data_dict) try: connection = read_engine.connect() result = connection.execute( 'SELECT 1 FROM "_table_metadata" WHERE name = %s AND alias_of IS NULL', new_data_dict['id'] ).fetchone() if result: new_data_dict['datastore_active'] = True else: new_data_dict['datastore_active'] = False finally: connection.close() return new_data_dict self.resource_show_action = new_resource_show
def configure(self, config): self.config = config # check for ckan.datastore.write_url and ckan.datastore.read_url if (not 'ckan.datastore.write_url' in config): error_msg = 'ckan.datastore.write_url not found in config' raise DatastoreException(error_msg) # Legacy mode means that we have no read url. Consequently sql search is not # available and permissions do not have to be changed. In legacy mode, the # datastore runs on PG prior to 9.0 (for example 8.4). self.legacy_mode = 'ckan.datastore.read_url' not in self.config # Check whether we are running one of the paster commands which means # that we should ignore the following tests. import sys if sys.argv[0].split('/')[-1] == 'paster' and 'datastore' in sys.argv[1:]: log.warn('Omitting permission checks because you are ' 'running paster commands.') return self.ckan_url = self.config['sqlalchemy.url'] self.write_url = self.config['ckan.datastore.write_url'] if self.legacy_mode: self.read_url = self.write_url else: self.read_url = self.config['ckan.datastore.read_url'] if model.engine_is_pg(): if not self._is_read_only_database(): # Make sure that the right permissions are set # so that no harmful queries can be made if not ('debug' in config and config['debug']): self._check_separate_db() if self.legacy_mode: log.warn('Legacy mode active. The sql search will not be available.') else: self._check_read_permissions() self._create_alias_table() else: log.warn("We detected that CKAN is running on a read only database. " "Permission checks and the creation of _table_metadata are skipped.") else: log.warn("We detected that you do not use a PostgreSQL database. " "The DataStore will NOT work and datastore tests will be skipped.") ## Do light wrapping around action function to add datastore_active ## to resource dict. Not using IAction extension as this prevents other plugins ## from having a custom resource_read. # Make sure actions are cached resource_show = p.toolkit.get_action('resource_show') def new_resource_show(context, data_dict): engine = db._get_engine( context, {'connection_url': self.read_url} ) new_data_dict = resource_show(context, data_dict) try: connection = engine.connect() result = connection.execute( 'SELECT 1 FROM "_table_metadata" WHERE name = %s AND alias_of IS NULL', new_data_dict['id'] ).fetchone() if result: new_data_dict['datastore_active'] = True else: new_data_dict['datastore_active'] = False finally: connection.close() return new_data_dict ## Make sure do not run many times if configure is called repeatedly ## as in tests. if not hasattr(resource_show, '_datastore_wrapped'): new_resource_show._datastore_wrapped = True logic._actions['resource_show'] = new_resource_show
def configure(self, config): self.config = config # check for ckan.datastore.write_url and ckan.datastore.read_url if (not 'ckan.datastore.write_url' in config): error_msg = 'ckan.datastore.write_url not found in config' raise DatastoreException(error_msg) # Legacy mode means that we have no read url. Consequently sql search is not # available and permissions do not have to be changed. In legacy mode, the # datastore runs on PG prior to 9.0 (for example 8.4). self.legacy_mode = 'ckan.datastore.read_url' not in self.config datapusher_formats = config.get('datapusher.formats', '').split() self.datapusher_formats = datapusher_formats or DEFAULT_FORMATS # Check whether we are running one of the paster commands which means # that we should ignore the following tests. if sys.argv[0].split('/')[-1] == 'paster' and 'datastore' in sys.argv[1:]: log.warn('Omitting permission checks because you are ' 'running paster commands.') return self.ckan_url = self.config['sqlalchemy.url'] self.write_url = self.config['ckan.datastore.write_url'] if self.legacy_mode: self.read_url = self.write_url log.warn('Legacy mode active. ' 'The sql search will not be available.') else: self.read_url = self.config['ckan.datastore.read_url'] read_engine = db._get_engine( {'connection_url': self.read_url}) if not model.engine_is_pg(read_engine): log.warn('We detected that you do not use a PostgreSQL ' 'database. The DataStore will NOT work and DataStore ' 'tests will be skipped.') return if self._is_read_only_database(): log.warn('We detected that CKAN is running on a read ' 'only database. Permission checks and the creation ' 'of _table_metadata are skipped.') else: self._check_urls_and_permissions() self._create_alias_table() # update the resource_show action to have datastore_active property if self.resource_show_action is None: resource_show = p.toolkit.get_action('resource_show') @logic.side_effect_free def new_resource_show(context, data_dict): new_data_dict = resource_show(context, data_dict) try: connection = read_engine.connect() result = connection.execute( 'SELECT 1 FROM "_table_metadata" WHERE name = %s AND alias_of IS NULL', new_data_dict['id'] ).fetchone() if result: new_data_dict['datastore_active'] = True else: new_data_dict['datastore_active'] = False finally: connection.close() return new_data_dict self.resource_show_action = new_resource_show
def configure(self, config): self.config = config # check for ckan.datastore.write_url and ckan.datastore.read_url if not "ckan.datastore.write_url" in config: error_msg = "ckan.datastore.write_url not found in config" raise DatastoreException(error_msg) # Legacy mode means that we have no read url. Consequently sql search is not # available and permissions do not have to be changed. In legacy mode, the # datastore runs on PG prior to 9.0 (for example 8.4). self.legacy_mode = "ckan.datastore.read_url" not in self.config # Check whether we are running one of the paster commands which means # that we should ignore the following tests. import sys if sys.argv[0].split("/")[-1] == "paster" and "datastore" in sys.argv[1:]: log.warn("Omitting permission checks because you are " "running paster commands.") return self.ckan_url = self.config["sqlalchemy.url"] self.write_url = self.config["ckan.datastore.write_url"] if self.legacy_mode: self.read_url = self.write_url log.warn("Legacy mode active. " "The sql search will not be available.") else: self.read_url = self.config["ckan.datastore.read_url"] if not model.engine_is_pg(): log.warn( "We detected that you do not use a PostgreSQL " "database. The DataStore will NOT work and DataStore " "tests will be skipped." ) return if self._is_read_only_database(): log.warn( "We detected that CKAN is running on a read " "only database. Permission checks and the creation " "of _table_metadata are skipped." ) else: self._check_urls_and_permissions() self._create_alias_table() ## Do light wrapping around action function to add datastore_active ## to resource dict. Not using IAction extension as this prevents ## other plugins from having a custom resource_show. # Make sure actions are cached resource_show = p.toolkit.get_action("resource_show") @logic.side_effect_free def new_resource_show(context, data_dict): engine = db._get_engine(context, {"connection_url": self.read_url}) new_data_dict = resource_show(context, data_dict) try: connection = engine.connect() result = connection.execute( 'SELECT 1 FROM "_table_metadata" WHERE name = %s AND alias_of IS NULL', new_data_dict["id"] ).fetchone() if result: new_data_dict["datastore_active"] = True else: new_data_dict["datastore_active"] = False finally: connection.close() return new_data_dict ## Make sure do not run many times if configure is called repeatedly ## as in tests. if not hasattr(resource_show, "_datastore_wrapped"): new_resource_show._datastore_wrapped = True logic._actions["resource_show"] = new_resource_show
context['schema'] = package_schema if status == 'new': # We need to explicitly provide a package ID, otherwise # ckanext-spatial won't be be able to link the extent to the # package. if not package_dict.get('id'): package_dict['id'] = unicode(uuid.uuid4()) package_schema['id'] = [unicode] # Save reference to the package on the object harvest_object.package_id = package_dict['id'] # Defer constraints and flush so the dataset can be indexed with # the harvest object id (on the after_show hook from the harvester # plugin) if model.engine_is_pg(): model.Session.execute( 'SET CONSTRAINTS harvest_object_package_id_fkey DEFERRED') model.Session.flush() if source_config.get('private_datasets', False): package_dict['private'] = True log.debug('package_create: %r', package_dict) try: package_dict_created = tk.get_action('package_create')( context.copy(), package_dict) log.info('Created new package name=%s id=%s guid=%s', package_dict.get('name'), package_dict_created['id'], harvest_object.guid) except tk.ValidationError, e: