def get(self, **kwargs): '''Return the result of a custom query. ''' params = external_common.parse_arguments(self.filters, kwargs) if not params.query: raise MissingArgumentError('query') try: query = json.loads(params.query) except ValueError: raise BadArgumentError( 'query', msg="Invalid JSON value for parameter 'query'" ) es = pyelasticsearch.ElasticSearch( urls=self.config.elasticsearch_urls, timeout=self.config.elasticsearch_timeout_extended, ) # Set indices. indices = [] if not params.indices: # By default, use the last two indices. today = utc_now() last_week = today - datetime.timedelta(days=7) indices = self.generate_list_of_indexes(last_week, today) elif len(params.indices) == 1 and params.indices[0] == 'ALL': # If we want all indices, just do nothing. pass else: indices = params.indices search_args = {} if indices: search_args['index'] = indices search_args['doc_type'] = self.config.elasticsearch_doctype try: results = es.search( query, **search_args ) except ElasticHttpNotFoundError, e: missing_index = re.findall(BAD_INDEX_REGEX, e.error)[0] raise ResourceNotFound( "elasticsearch index '%s' does not exist" % missing_index )
def get(self, **kwargs): self.context.logger.info('Running %s' % self.__class__.__name__) raise ResourceNotFound('not here')
def update_field(self, **kwargs): """Update an existing field in the database. If the field does not exist yet, a ResourceNotFound error is raised. If you want to update only some keys, just do not pass the ones you don't want to change. """ filters = [ ('name', None, 'str'), ('data_validation_type', None, 'str'), ('default_value', None, 'str'), ('description', None, 'str'), ('form_field_choices', None, ['list', 'str']), ('has_full_version', None, 'bool'), ('in_database_name', None, 'str'), ('is_exposed', None, 'bool'), ('is_returned', None, 'bool'), ('is_mandatory', None, 'bool'), ('query_type', None, 'str'), ('namespace', None, 'str'), ('permissions_needed', None, ['list', 'str']), ('storage_mapping', None, 'json'), ] params = external_common.parse_arguments(filters, kwargs) if not params['name']: raise MissingArgumentError('name') # Remove all the parameters that were not explicitely passed. for key in params.keys(): if key not in kwargs: del params[key] es_connection = self.get_connection() es_index = self.config.elasticsearch.elasticsearch_default_index es_doc_type = 'supersearch_fields' # First verify that the field does exist. try: old_value = es_connection.get( index=es_index, doc_type=es_doc_type, id=params['name'], )['_source'] # Only the actual document is of interest. except elasticsearch.exceptions.NotFoundError: # This field does not exist yet, it thus cannot be updated! raise ResourceNotFound( 'The field "%s" does not exist in the database, it needs to ' 'be created before it can be updated. ' % params['name'] ) # Then, if necessary, verify the new mapping. if ( ( 'storage_mapping' in params and params['storage_mapping'] != old_value['storage_mapping'] ) or ( 'in_database_name' in params and params['in_database_name'] != old_value['in_database_name'] ) ): # This is a change that will have an impact on the Elasticsearch # mapping, we first need to make sure it doesn't break. new_mapping = self.get_mapping(overwrite_mapping=params) # Try the mapping. If there is an error, an exception will be # raised. If an exception is raised, the new mapping will be # rejected. self.test_mapping(new_mapping) if ( 'storage_mapping' in params and params['storage_mapping'] != old_value['storage_mapping'] ): # The storage mapping is an object, and thus is treated # differently than other fields by Elasticsearch. If a user # changes the object by removing a field from it, that field won't # be removed as part of the update (which performs a merge of all # objects in the back-end). We therefore want to perform the merge # ourselves, and remove the field from the database before # re-indexing it. new_doc = old_value.copy() new_doc.update(params) es_connection.delete( index=es_index, doc_type=es_doc_type, id=new_doc['name'], ) es_connection.index( index=es_index, doc_type=es_doc_type, body=new_doc, id=new_doc['name'], op_type='create', refresh=True, ) # If we made a change to the storage_mapping, log that change. self.config.logger.info( 'Elasticsearch mapping changed for field "%s", ' 'was "%s", now "%s"', params['name'], old_value['storage_mapping'], new_doc['storage_mapping'], ) else: # Then update the new field in the database. Note that # Elasticsearch takes care of merging the new document into the # old one, so missing values won't be changed. es_connection.update( index=es_index, doc_type=es_doc_type, body={'doc': params}, id=params['name'], refresh=True, ) return True
def update_field(self, **kwargs): """Update an existing field in the database. If the field does not exist yet, a ResourceNotFound error is raised. If you want to update only some keys, just do not pass the ones you don't want to change. """ filters = [ ('name', None, 'str'), ('data_validation_type', None, 'str'), ('default_value', None, 'str'), ('description', None, 'str'), ('form_field_choices', None, ['list', 'str']), ('has_full_version', None, 'bool'), ('in_database_name', None, 'str'), ('is_exposed', None, 'bool'), ('is_returned', None, 'bool'), ('is_mandatory', None, 'bool'), ('query_type', None, 'str'), ('namespace', None, 'str'), ('permissions_needed', None, ['list', 'str']), ('storage_mapping', None, 'json'), ] params = external_common.parse_arguments(filters, kwargs) if not params['name']: raise MissingArgumentError('name') # Remove all the parameters that were not explicitely passed. for key in params.keys(): if key not in kwargs: del params[key] # Before making the change, make sure it does not break indexing. new_mapping = self.get_mapping(overwrite_mapping=params) # Try the mapping. If there is an error, an exception will be raised. # If an exception is raised, the new mapping will be rejected. self.test_mapping(new_mapping) es_connection = self.get_connection().get_es() # First verify that the field does exist. try: old_value = es_connection.get( index=self.config.elasticsearch_default_index, doc_type='supersearch_fields', id=params['name'], )['_source'] # Only the actual document is of interest. except ElasticHttpNotFoundError: # This field does not exist yet, it thus cannot be updated! raise ResourceNotFound( 'The field "%s" does not exist in the database, it needs to ' 'be created before it can be updated. ' % params['name']) if 'storage_mapping' in params and old_value['storage_mapping']: # The storage mapping is an object, and thus is treated # differently than other fields by Elasticsearch. If a user # changes the object by removing a field from it, that field won't # be removed as part of the update (which performs a merge of all # objects in the back-end). We therefore want to remove that field # before we do the merge, so that it is entirely overwritten. es_connection.update( index=self.config.elasticsearch_default_index, doc_type='supersearch_fields', script='ctx._source.remove("storage_mapping")', id=params['name'], refresh=True, ) # Then update the new field in the database. Note that pyelasticsearch # takes care of merging the new document into the old one, so missing # values won't be changed. es_connection.update( index=self.config.elasticsearch_default_index, doc_type='supersearch_fields', doc=params, id=params['name'], refresh=True, ) if 'storage_mapping' in params: # If we made a change to the storage_mapping, log that change. self.config.logger.info( 'elasticsearch mapping changed for field "%s", ' 'was "%s", now "%s"', params['name'], old_value['storage_mapping'], params['storage_mapping'], ) return True
def update_field(self, **kwargs): """Update an existing field in the database. If the field does not exist yet, a ResourceNotFound error is raised. If you want to update only some keys, just do not pass the ones you don't want to change. """ filters = [ ('name', None, 'str'), ('data_validation_type', None, 'str'), ('default_value', None, 'str'), ('description', None, 'str'), ('form_field_type', None, 'str'), ('form_field_choices', None, ['list', 'str']), ('has_full_version', None, 'bool'), ('in_database_name', None, 'str'), ('is_exposed', None, 'bool'), ('is_returned', None, 'bool'), ('is_mandatory', None, 'bool'), ('query_type', None, 'str'), ('namespace', None, 'str'), ('permissions_needed', None, ['list', 'str']), ('storage_mapping', None, 'json'), ] params = external_common.parse_arguments(filters, kwargs) if not params['name']: raise MissingArgumentError('name') # Remove all the parameters that were not explicitely passed. for key in params.keys(): if key not in kwargs: del params[key] es_connection = self.get_connection().get_es() # First verify that the field does exist. try: es_connection.get( index=self.config.elasticsearch_default_index, doc_type='supersearch_fields', id=params['name'], ) except ElasticHttpNotFoundError: # This field does not exist yet, it thus cannot be updated! raise ResourceNotFound( 'The field "%s" does not exist in the database, it needs to ' 'be created before it can be updated. ' % params['name'] ) # Then update the new field in the database. Note that pyelasticsearch # takes care of merging the new document into the old one, so missing # values won't be changed. es_connection.update( index=self.config.elasticsearch_default_index, doc_type='supersearch_fields', doc=params, id=params['name'], refresh=True, ) return True