def datastore_delete(context, data_dict): '''Deletes a table or a set of records from the DataStore. :param resource_id: resource id that the data will be deleted from. (optional) :type resource_id: string :param force: set to True to edit a read-only resource :type force: bool (optional, default: False) :param filters: filters to apply before deleting (eg {"name": "fred"}). If missing delete whole table and all dependent views. (optional) :type filters: dictionary **Results:** :returns: Original filters sent. :rtype: dictionary ''' schema = context.get('schema', dsschema.datastore_upsert_schema()) filters = data_dict.pop('filters', None) data_dict, errors = _validate(data_dict, schema, context) if filters: data_dict['filters'] = filters if errors: raise p.toolkit.ValidationError(errors) p.toolkit.check_access('datastore_delete', context, data_dict) if not data_dict.pop('force', False): resource_id = data_dict['resource_id'] _check_read_only(context, resource_id) data_dict['connection_url'] = pylons.config['ckan.datastore.write_url'] res_id = data_dict['resource_id'] resources_sql = sqlalchemy.text(u'''SELECT 1 FROM "_table_metadata" WHERE name = :id AND alias_of IS NULL''' ) results = db._get_engine(data_dict).execute(resources_sql, id=res_id) res_exists = results.rowcount > 0 if not res_exists: raise p.toolkit.ObjectNotFound( p.toolkit._(u'Resource "{0}" was not found.'.format(res_id))) result = db.delete(context, data_dict) # Set the datastore_active flag on the resource if necessary if not data_dict.get('filters'): p.toolkit.get_action('resource_patch')(context, { 'id': data_dict['resource_id'], 'datastore_active': False }) result.pop('id', None) result.pop('connection_url') return result
def datastore_delete(context, data_dict): '''Deletes a table or a set of records from the DataStore. :param resource_id: resource id that the data will be deleted from. (optional) :type resource_id: string :param force: set to True to edit a read-only resource :type force: bool (optional, default: False) :param filters: filters to apply before deleting (eg {"name": "fred"}). If missing delete whole table and all dependent views. (optional) :type filters: dictionary **Results:** :returns: Original filters sent. :rtype: dictionary ''' schema = context.get('schema', dsschema.datastore_upsert_schema()) filters = data_dict.pop('filters', None) data_dict, errors = _validate(data_dict, schema, context) if filters: data_dict['filters'] = filters if errors: raise p.toolkit.ValidationError(errors) p.toolkit.check_access('datastore_delete', context, data_dict) if not data_dict.pop('force', False): resource_id = data_dict['resource_id'] _check_read_only(context, resource_id) data_dict['connection_url'] = pylons.config['ckan.datastore.write_url'] res_id = data_dict['resource_id'] resources_sql = sqlalchemy.text(u'''SELECT 1 FROM "_table_metadata" WHERE name = :id AND alias_of IS NULL''') results = db._get_engine(data_dict).execute(resources_sql, id=res_id) res_exists = results.rowcount > 0 if not res_exists: raise p.toolkit.ObjectNotFound(p.toolkit._( u'Resource "{0}" was not found.'.format(res_id) )) result = db.delete(context, data_dict) # Set the datastore_active flag on the resource if necessary if not data_dict.get('filters'): p.toolkit.get_action('resource_patch')( context, {'id': data_dict['resource_id'], 'datastore_active': False}) result.pop('id', None) result.pop('connection_url') return result
def datastore_delete(context, data_dict): """Deletes a table or a set of records from the DataStore. :param resource_id: resource id that the data will be deleted from. (optional) :type resource_id: string :param force: set to True to edit a read-only resource :type force: bool (optional, default: False) :param filters: filters to apply before deleting (eg {"name": "fred"}). If missing delete whole table and all dependent views. (optional) :type filters: dictionary **Results:** :returns: Original filters sent. :rtype: dictionary """ schema = context.get("schema", dsschema.datastore_upsert_schema()) filters = data_dict.pop("filters", None) data_dict, errors = _validate(data_dict, schema, context) if filters: data_dict["filters"] = filters if errors: raise p.toolkit.ValidationError(errors) p.toolkit.check_access("datastore_delete", context, data_dict) if not data_dict.pop("force", False): resource_id = data_dict["resource_id"] _check_read_only(context, resource_id) data_dict["connection_url"] = pylons.config["ckan.datastore.write_url"] res_id = data_dict["resource_id"] resources_sql = sqlalchemy.text( u"""SELECT 1 FROM "_table_metadata" WHERE name = :id AND alias_of IS NULL""" ) results = db._get_engine(data_dict).execute(resources_sql, id=res_id) res_exists = results.rowcount > 0 if not res_exists: raise p.toolkit.ObjectNotFound(p.toolkit._(u'Resource "{0}" was not found.'.format(res_id))) result = db.delete(context, data_dict) result.pop("id", None) result.pop("connection_url") return result
def audited_datastore_update(context, data_dict=None): logic.check_access('audited_datastore_update', context, data_dict) log.debug('starting: audited_datastore_update') # most of this copy pasted from datastore_upsert schema = context.get('schema', dsschema.datastore_upsert_schema()) schema.pop('__junk', None) records = data_dict.pop('records', []) update_time = data_dict.pop(UPDATE_TIMESTAMP_FIELD, str(datetime.utcnow())) delete_absent = data_dict.pop('delete_absent', True) if re.compile('[+-]\d\d:\d\d$').search(update_time): update_time = to_timestamp_naive(update_time) data_dict, errors = _validate(data_dict, schema, context) if errors: raise tk.ValidationError(errors) if not data_dict.pop('force', False): if get_ckan_version(context) >= u"2.3": resource_id = data_dict['resource_id'] _check_read_only(context, resource_id) else: _check_read_only(context, data_dict) data_dict['connection_url'] = pylons.config['ckan.datastore.write_url'] res_id = data_dict['resource_id'] resources_sql = sqlalchemy.text(u'''SELECT 1 FROM "_table_metadata" WHERE name = :id AND alias_of IS NULL''' ) results = db._get_engine(data_dict).execute(resources_sql, id=res_id) res_exists = results.rowcount > 0 if not res_exists: raise tk.ObjectNotFound( tk._(u'Resource "{0}" was not found.'.format(res_id))) result = do_audit(context, data_dict, records, update_time, delete_absent) result.pop('id', None) result.pop('records', None) result.pop('connection_url', None) return result
def datastore_delete(context, data_dict): '''Deletes a table or a set of records from the DataStore. :param resource_id: resource id that the data will be deleted from. (optional) :type resource_id: string :param force: set to True to edit a read-only resource :type force: bool (optional, default: False) :param filters: filters to apply before deleting (eg {"name": "fred"}). If missing delete whole table and all dependent views. (optional) :type filters: dictionary **Results:** :returns: Original filters sent. :rtype: dictionary ''' schema = context.get('schema', dsschema.datastore_upsert_schema())
def datastore_upsert(context, data_dict): '''Updates or inserts into a table in the DataStore The datastore_upsert API action allows you to add or edit records to an existing DataStore resource. In order for the *upsert* and *update* methods to work, a unique key has to be defined via the datastore_create action. The available methods are: *upsert* Update if record with same key already exists, otherwise insert. Requires unique key. *insert* Insert only. This method is faster that upsert, but will fail if any inserted record matches an existing one. Does *not* require a unique key. *update* Update only. An exception will occur if the key that should be updated does not exist. Requires unique key. :param resource_id: resource id that the data is going to be stored under. :type resource_id: string :param force: set to True to edit a read-only resource :type force: bool (optional, default: False) :param records: the data, eg: [{"dob": "2005", "some_stuff": ["a","b"]}] (optional) :type records: list of dictionaries :param method: the method to use to put the data into the datastore. Possible options are: upsert, insert, update (optional, default: upsert) :type method: string **Results:** :returns: The modified data object. :rtype: dictionary ''' schema = context.get('schema', dsschema.datastore_upsert_schema()) records = data_dict.pop('records', None) data_dict, errors = _validate(data_dict, schema, context) if records: data_dict['records'] = records if errors: raise p.toolkit.ValidationError(errors) p.toolkit.check_access('datastore_upsert', context, data_dict) if not data_dict.pop('force', False): resource_id = data_dict['resource_id'] _check_read_only(context, resource_id) data_dict['connection_url'] = pylons.config['ckan.datastore.write_url'] res_id = data_dict['resource_id'] resources_sql = sqlalchemy.text(u'''SELECT 1 FROM "_table_metadata" WHERE name = :id AND alias_of IS NULL''' ) results = db._get_engine(data_dict).execute(resources_sql, id=res_id) res_exists = results.rowcount > 0 if not res_exists: raise p.toolkit.ObjectNotFound( p.toolkit._(u'Resource "{0}" was not found.'.format(res_id))) result = db.upsert(context, data_dict) result.pop('id', None) result.pop('connection_url') return result
def datastore_upsert(context, data_dict): '''Updates or inserts into a table in the DataStore The datastore_upsert API action allows you to add or edit records to an existing DataStore resource. In order for the *upsert* and *update* methods to work, a unique key has to be defined via the datastore_create action. The available methods are: *upsert* Update if record with same key already exists, otherwise insert. Requires unique key. *insert* Insert only. This method is faster that upsert, but will fail if any inserted record matches an existing one. Does *not* require a unique key. *update* Update only. An exception will occur if the key that should be updated does not exist. Requires unique key. :param resource_id: resource id that the data is going to be stored under. :type resource_id: string :param force: set to True to edit a read-only resource :type force: bool (optional, default: False) :param records: the data, eg: [{"dob": "2005", "some_stuff": ["a","b"]}] (optional) :type records: list of dictionaries :param method: the method to use to put the data into the datastore. Possible options are: upsert, insert, update (optional, default: upsert) :type method: string **Results:** :returns: The modified data object. :rtype: dictionary ''' schema = context.get('schema', dsschema.datastore_upsert_schema()) records = data_dict.pop('records', None) data_dict, errors = _validate(data_dict, schema, context) if records: data_dict['records'] = records if errors: raise p.toolkit.ValidationError(errors) p.toolkit.check_access('datastore_upsert', context, data_dict) if not data_dict.pop('force', False): resource_id = data_dict['resource_id'] _check_read_only(context, resource_id) data_dict['connection_url'] = pylons.config['ckan.datastore.write_url'] res_id = data_dict['resource_id'] resources_sql = sqlalchemy.text(u'''SELECT 1 FROM "_table_metadata" WHERE name = :id AND alias_of IS NULL''') results = db._get_engine(data_dict).execute(resources_sql, id=res_id) res_exists = results.rowcount > 0 if not res_exists: raise p.toolkit.ObjectNotFound(p.toolkit._( u'Resource "{0}" was not found.'.format(res_id) )) result = db.upsert(context, data_dict) result.pop('id', None) result.pop('connection_url') return result
def datastore_upsert(context, data_dict): '''Updates or inserts into a table in the DataStore The datastore_upsert API action allows you to add or edit records to an existing DataStore resource. In order for the *upsert* and *update* methods to work, a unique key has to be defined via the datastore_create action. The available methods are: *upsert* Update if record with same key already exists, otherwise insert. Requires unique key. *insert* Insert only. This method is faster that upsert, but will fail if any inserted record matches an existing one. Does *not* require a unique key. *update* Update only. An exception will occur if the key that should be updated does not exist. Requires unique key. :param resource_id: resource id that the data is going to be stored under. :type resource_id: string :param force: set to True to edit a read-only resource :type force: bool (optional, default: False) :param records: the data, eg: [{"dob": "2005", "some_stuff": ["a","b"]}] (optional) :type records: list of dictionaries :param method: the method to use to put the data into the datastore. Possible options are: upsert, insert, update (optional, default: upsert) :type method: string :param calculate_record_count: updates the stored count of records, used to optimize datastore_search in combination with the `total_estimation_threshold` parameter. If doing a series of requests to change a resource, you only need to set this to True on the last request. :type calculate_record_count: bool (optional, default: False) :param dry_run: set to True to abort transaction instead of committing, e.g. to check for validation or type errors. :type dry_run: bool (optional, default: False) **Results:** :returns: The modified data object. :rtype: dictionary ''' backend = DatastoreBackend.get_active_backend() schema = context.get('schema', dsschema.datastore_upsert_schema()) records = data_dict.pop('records', None) data_dict, errors = _validate(data_dict, schema, context) if records: data_dict['records'] = records if errors: raise p.toolkit.ValidationError(errors) p.toolkit.check_access('datastore_upsert', context, data_dict) resource_id = data_dict['resource_id'] if not data_dict.pop('force', False): _check_read_only(context, resource_id) res_exists = backend.resource_exists(resource_id) if not res_exists: raise p.toolkit.ObjectNotFound( p.toolkit._(u'Resource "{0}" was not found.'.format(resource_id))) result = backend.upsert(context, data_dict) result.pop('id', None) result.pop('connection_url', None) if data_dict.get('calculate_record_count', False): backend.calculate_record_count(data_dict['resource_id']) return result
def datastore_delete(context, data_dict): '''Deletes a table or a set of records from the DataStore. :param resource_id: resource id that the data will be deleted from. (optional) :type resource_id: string :param force: set to True to edit a read-only resource :type force: bool (optional, default: False) :param filters: filters to apply before deleting (eg {"name": "fred"}). If missing delete whole table and all dependent views. (optional) :type filters: dictionary **Results:** :returns: Original filters sent. :rtype: dictionary ''' schema = context.get('schema', dsschema.datastore_upsert_schema()) backend = DatastoreBackend.get_active_backend() # Remove any applied filters before running validation. filters = data_dict.pop('filters', None) data_dict, errors = _validate(data_dict, schema, context) if filters is not None: if not isinstance(filters, dict): raise p.toolkit.ValidationError( {'filters': ['filters must be either a dict or null.']}) data_dict['filters'] = filters if errors: raise p.toolkit.ValidationError(errors) p.toolkit.check_access('datastore_delete', context, data_dict) if not data_dict.pop('force', False): resource_id = data_dict['resource_id'] _check_read_only(context, resource_id) res_id = data_dict['resource_id'] res_exists = backend.resource_exists(res_id) if not res_exists: raise p.toolkit.ObjectNotFound( p.toolkit._(u'Resource "{0}" was not found.'.format(res_id))) result = backend.delete(context, data_dict) # Set the datastore_active flag on the resource if necessary model = _get_or_bust(context, 'model') resource = model.Resource.get(data_dict['resource_id']) if (not data_dict.get('filters') and resource.extras.get('datastore_active') is True): log.debug('Setting datastore_active=False on resource {0}'.format( resource.id)) set_datastore_active_flag(model, data_dict, False) result.pop('id', None) result.pop('connection_url', None) return result
def datastore_delete(context, data_dict): '''Deletes a table or a set of records from the DataStore. :param resource_id: resource id that the data will be deleted from. (optional) :type resource_id: string :param force: set to True to edit a read-only resource :type force: bool (optional, default: False) :param filters: filters to apply before deleting (eg {"name": "fred"}). If missing delete whole table and all dependent views. (optional) :type filters: dictionary **Results:** :returns: Original filters sent. :rtype: dictionary ''' schema = context.get('schema', dsschema.datastore_upsert_schema()) backend = DatastoreBackend.get_active_backend() # Remove any applied filters before running validation. filters = data_dict.pop('filters', None) data_dict, errors = _validate(data_dict, schema, context) if filters is not None: if not isinstance(filters, dict): raise p.toolkit.ValidationError({ 'filters': [ 'filters must be either a dict or null.' ] }) data_dict['filters'] = filters if errors: raise p.toolkit.ValidationError(errors) p.toolkit.check_access('datastore_delete', context, data_dict) if not data_dict.pop('force', False): resource_id = data_dict['resource_id'] _check_read_only(context, resource_id) res_id = data_dict['resource_id'] res_exists = backend.resource_exists(res_id) if not res_exists: raise p.toolkit.ObjectNotFound(p.toolkit._( u'Resource "{0}" was not found.'.format(res_id) )) result = backend.delete(context, data_dict) # Set the datastore_active flag on the resource if necessary model = _get_or_bust(context, 'model') resource = model.Resource.get(data_dict['resource_id']) if (not data_dict.get('filters') and resource is not None and resource.extras.get('datastore_active') is True): log.debug( 'Setting datastore_active=False on resource {0}'.format( resource.id) ) set_datastore_active_flag(model, data_dict, False) result.pop('id', None) result.pop('connection_url', None) return result
def datastore_upsert(context, data_dict): '''Updates or inserts into a table in the DataStore The datastore_upsert API action allows you to add or edit records to an existing DataStore resource. In order for the *upsert* and *update* methods to work, a unique key has to be defined via the datastore_create action. The available methods are: *upsert* Update if record with same key already exists, otherwise insert. Requires unique key. *insert* Insert only. This method is faster that upsert, but will fail if any inserted record matches an existing one. Does *not* require a unique key. *update* Update only. An exception will occur if the key that should be updated does not exist. Requires unique key. :param resource_id: resource id that the data is going to be stored under. :type resource_id: string :param force: set to True to edit a read-only resource :type force: bool (optional, default: False) :param records: the data, eg: [{"dob": "2005", "some_stuff": ["a","b"]}] (optional) :type records: list of dictionaries :param method: the method to use to put the data into the datastore. Possible options are: upsert, insert, update (optional, default: upsert) :type method: string :param dry_run: set to True to abort transaction instead of committing, e.g. to check for validation or type errors. :type dry_run: bool (optional, default: False) **Results:** :returns: The modified data object. :rtype: dictionary ''' backend = DatastoreBackend.get_active_backend() schema = context.get('schema', dsschema.datastore_upsert_schema()) records = data_dict.pop('records', None) data_dict, errors = _validate(data_dict, schema, context) if records: data_dict['records'] = records if errors: raise p.toolkit.ValidationError(errors) p.toolkit.check_access('datastore_upsert', context, data_dict) resource_id = data_dict['resource_id'] if not data_dict.pop('force', False): _check_read_only(context, resource_id) res_exists = backend.resource_exists(resource_id) if not res_exists: raise p.toolkit.ObjectNotFound(p.toolkit._( u'Resource "{0}" was not found.'.format(resource_id) )) result = backend.upsert(context, data_dict) result.pop('id', None) result.pop('connection_url', None) return result