def test_repair_missing_markers_on_non_default_namespace(self): self.i3 = TestModel.objects.using("ns1").create(id=self.i1.pk, name="name1", counter1=1, counter2=1) self.i4 = TestModel.objects.using("ns1").create(id=self.i2.pk, name="name3", counter1=1, counter2=2) NS1_NAMESPACE = settings.DATABASES["ns1"]["NAMESPACE"] instance_key = datastore.Key.from_path(TestModel._meta.db_table, self.i2.pk, namespace=DEFAULT_NAMESPACE) instance_key_ns1 = datastore.Key.from_path(TestModel._meta.db_table, self.i2.pk, namespace=NS1_NAMESPACE) marker = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i2.name).hexdigest()) marker_key_default = datastore.Key.from_path(UniqueMarker.kind(), marker, namespace=DEFAULT_NAMESPACE) marker_key_ns1 = datastore.Key.from_path(UniqueMarker.kind(), marker, namespace=NS1_NAMESPACE) datastore.Delete(marker_key_ns1) datastore.Delete(marker_key_default) UniqueAction.objects.create(action_type="repair", model=encode_model(TestModel), db="ns1") process_task_queues() a = UniqueAction.objects.get() self.assertEqual(a.status, "done") # Is the missing marker for the default namespace left alone? self.assertRaises(datastore_errors.EntityNotFoundError, datastore.Get, marker_key_default) # Is the missing marker restored? marker = datastore.Get(marker_key_ns1) self.assertTrue(marker) self.assertTrue(isinstance(marker["instance"], datastore.Key)) self.assertEqual(instance_key_ns1, marker["instance"]) self.assertTrue(marker["created"])
def execute(self): table = self.table query = datastore.Query(table, keys_only=True, namespace=self.namespace) while query.Count(): datastore.Delete(query.Run()) # Delete the markers we need to from djangae.db.constraints import UniqueMarker query = datastore.Query(UniqueMarker.kind(), keys_only=True, namespace=self.namespace) query["__key__ >="] = datastore.Key.from_path(UniqueMarker.kind(), self.table, namespace=self.namespace) query["__key__ <"] = datastore.Key.from_path(UniqueMarker.kind(), u"{}{}".format( self.table, u'\ufffd'), namespace=self.namespace) while query.Count(): datastore.Delete(query.Run()) # TODO: ideally we would only clear the cached objects for the table that was flushed, but # we have no way of doing that memcache.flush_all() caching.get_context().reset()
def map(entity, model, *args, **kwargs): """ The Clean mapper maps over all UniqueMarker instances. """ alias = kwargs.get("db", "default") namespace = settings.DATABASES.get(alias, {}).get("NAMESPACE", "") model = decode_model(model) if not entity.key().id_or_name().startswith(model._meta.db_table + "|"): # Only include markers which are for this model return assert namespace == entity.namespace() with disable_cache(): # At this point, the entity is a unique marker that is linked to an instance of 'model', now we should see if that instance exists! instance_id = entity["instance"].id_or_name() try: instance = model.objects.using(alias).get(pk=instance_id) except model.DoesNotExist: logger.info("Deleting unique marker %s because the associated instance no longer exists", entity.key().id_or_name()) datastore.Delete(entity) return # Get the possible unique markers for the entity, if this one doesn't exist in that list then delete it instance_entity, _ = django_instance_to_entities(connections[alias], instance._meta.fields, raw=True, instance=instance, check_null=False) identifiers = unique_identifiers_from_entity(model, instance_entity, ignore_pk=True) identifier_keys = [datastore.Key.from_path(UniqueMarker.kind(), i, namespace=entity["instance"].namespace()) for i in identifiers] if entity.key() not in identifier_keys: logger.info("Deleting unique marker %s because the it no longer represents the associated instance state", entity.key().id_or_name()) datastore.Delete(entity)
def DeleteBlob(self, blob_key): """Delete blob data from the datastore. Args: blob_key: Blob-key of existing blob to delete. Raises: ApplicationError: When a blob is not found or unable to be read. """ blob_info_key = datastore.Key.from_path(blobstore.BLOB_INFO_KIND, str(blob_key), namespace='') try: blob_info = datastore.Get(blob_info_key) except datastore_errors.EntityNotFoundError: raise apiproxy_errors.ApplicationError( blobstore_service_pb.BlobstoreServiceError.BLOB_NOT_FOUND) block_count = blob_info["size"] / blobstore.MAX_BLOB_FETCH_SIZE block_set = [] try: while block_count >= 0: entity = datastore.Entity(_BLOB_CHUNK_KIND_, name=str(blob_key) + "__" + str(block_count), namespace='') block_set.append(entity) block_count -= 1 datastore.Delete(block_set) datastore.Delete(blob_info_key) except: raise apiproxy_errors.ApplicationError( blobstore_service_pb.BlobstoreServiceError.BLOB_NOT_FOUND)
def DeleteBlob(cls, blobkey, storage): """Delete a blob. Args: blobkey: blobkey in str. storage: blobstore storage stub. """ datastore.Delete(cls.ToDatastoreBlobKey(blobkey)) blobinfo = datastore_types.Key.from_path(blobstore.BLOB_INFO_KIND, blobkey, namespace='') datastore.Delete(blobinfo) storage.DeleteBlob(blobkey)
def execute(self): self.select.execute() # This is a little bit more inefficient than just doing a keys_only query and # sending it to delete, but I think this is the sacrifice to make for the unique caching layer keys = [] def spawn_query(kind, key): qry = Query( kind, namespace=key.namespace() or None ) # TODO: is the namespace necessary if we're passing the key? qry["__key__ ="] = key return qry queries = [ spawn_query(x.key().kind(), x.key()) for x in self.select.results ] if not queries: return for entity in QueryByKeys(self.model, queries, [], self.namespace).Run(): keys.append(entity.key()) # Delete constraints if that's enabled if constraints.constraint_checks_enabled(self.model): constraints.release(self.model, entity) caching.remove_entities_from_cache_by_key(keys, self.namespace) datastore.Delete(keys)
def testDatastoreTypes(self): """Puts and gets different basic datastore types.""" entity = datastore.Entity('TestKind') entity.update({ 'rating': datastore_types.Rating(1), 'category': datastore_types.Category('bugs'), 'key': datastore_types.Key.from_path('foo', 'bar'), 'user': users.User('*****@*****.**'), 'text': datastore_types.Text('some text'), 'blob': datastore_types.Blob('data'), 'bytestring': datastore_types.ByteString('data'), 'im': datastore_types.IM('http://example.com/', 'Larry97'), 'geopt': datastore_types.GeoPt(1.1234, -1.1234), 'email': datastore_types.Email('*****@*****.**'), 'blobkey': datastore_types.BlobKey('27f5a7'), }) datastore.Put(entity) e = datastore.Get(entity) datastore.Delete(entity)
def test_clean_removes_markers_with_different_values_on_non_default_namespace(self): self.i3 = TestModel.objects.using("ns1").create(id=self.i1.pk, name="name1", counter1=1, counter2=1) self.i4 = TestModel.objects.using("ns1").create(id=self.i2.pk, name="name3", counter1=1, counter2=2) NS1_NAMESPACE = settings.DATABASES["ns1"]["NAMESPACE"] marker1 = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i3.name).hexdigest()) marker_key = datastore.Key.from_path(UniqueMarker.kind(), marker1, namespace=NS1_NAMESPACE) default_key = datastore.Key.from_path(UniqueMarker.kind(), marker1, namespace=DEFAULT_NAMESPACE) original_marker = datastore.Get(marker_key) default_marker = datastore.Get(default_key) marker2 = "{}|name:{}".format(TestModel._meta.db_table, md5("bananas").hexdigest()) new_marker = datastore.Entity(UniqueMarker.kind(), name=marker2, namespace=NS1_NAMESPACE) new_marker.update(original_marker) datastore.Put(new_marker) # This allows us to test: 1) namespaced markers will check against their namespace models (not all of them)" self.i1.delete() #... 2) the mapper only cleans the desired namespace datastore.Put(default_marker) UniqueAction.objects.create(action_type="clean", model=encode_model(TestModel), db="ns1") process_task_queues() self.assertRaises(datastore_errors.EntityNotFoundError, datastore.Get, new_marker.key()) self.assertTrue(datastore.Get(default_marker.key())) self.assertTrue(datastore.Get(marker_key)) datastore.Delete(default_marker)
def post(self): kind = self.request.get('kind') keys = [] index = 0 num_keys = int(self.request.get('numkeys')) for i in xrange(1, num_keys + 1): key = self.request.get('key%d' % i) if key: keys.append(key) if self.request.get('action') == 'Delete': num_deleted = 0 for key in keys: datastore.Delete(datastore.Key(key)) num_deleted = num_deleted + 1 message = '%d entit%s deleted.' % (num_deleted, ('ies', 'y')[num_deleted == 1]) self.redirect( '%s&msg=%s' % (self.request.get('next'), urllib.quote_plus(message))) return self.error(404)
def flush(self): """Helper function to remove the current datastore and re-open the stubs""" if self.remote: import random, string code = ''.join([random.choice(string.ascii_letters) for x in range(4)]) print '\n\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!' print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!' print "Warning! You're about to delete the *production* datastore!" print 'Only models defined in your INSTALLED_APPS can be removed!' print 'If you want to clear the whole datastore you have to use the ' \ 'datastore viewer in the dashboard. Also, in order to delete all '\ 'unneeded indexes you have to run appcfg.py vacuum_indexes.' print 'In order to proceed you have to enter the following code:' print code response = raw_input('Repeat: ') if code == response: print 'Deleting...' from django.db import models from google.appengine.api import datastore as ds for model in models.get_models(): print 'Deleting %s...' % model._meta.db_table while True: data = ds.Query(model._meta.db_table, keys_only=True).Get(200) if not data: break ds.Delete(data) print "Datastore flushed! Please check your dashboard's " \ 'datastore viewer for any remaining entities and remove ' \ 'all unneeded indexes with manage.py vacuum_indexes.' else: print 'Aborting' exit() else: destroy_datastore(*self._get_paths()) self._setup_stubs()
def execute(self): table = self.table query = datastore.Query(table, keys_only=True) while query.Count(): datastore.Delete(query.Run()) # Delete the markers we need to from djangae.db.constraints import UniqueMarker query = datastore.Query(UniqueMarker.kind(), keys_only=True) query["__key__ >="] = datastore.Key.from_path(UniqueMarker.kind(), self.table) query["__key__ <"] = datastore.Key.from_path(UniqueMarker.kind(), u"{}{}".format(self.table, u'\ufffd')) while query.Count(): datastore.Delete(query.Run()) cache.clear() clear_context_cache()
def __Increment(self, nodes_with_children, score_entities, score_entities_to_delete): """Changes child counts for given nodes. This method will create nodes as needed. Args: nodes_with_children: A dict of (node_key, child) tuples to deltas score_entities: Additional score entities to persist as part of this transaction Returns: None """ keys = list(set(key for ((key, _), delta) in nodes_with_children.iteritems() if delta != 0)) if not keys: return # Nothing to do nodes = datastore.Get(keys) node_dict = {} for (key, node) in zip(keys, nodes): if not node: node = datastore.Entity("ranker_node", parent=self.rootkey, name=key.name()) node["child_counts"] = [0] * self.branching_factor node_dict[key] = node for ((key, child), amount) in nodes_with_children.iteritems(): if amount != 0: node = node_dict[key] node["child_counts"][child] += amount assert node["child_counts"][child] >= 0 datastore.Put(node_dict.values() + score_entities) if score_entities_to_delete: datastore.Delete(score_entities_to_delete)
def execute(self): table = self.table query = datastore.Query(table, keys_only=True) while query.Count(): datastore.Delete(query.Run()) cache.clear()
def Delete(keys, **kwargs): """ Deletes one or more entities from the data store. :warning: Permanently deletes entities, use with care! Deletes the given entity or entities from the data store. You can only delete entities from your app. If there is an error, the function raises a subclass of :exc:`datastore_errors.Error`. :param keys: Key, str or list of keys or strings to be deleted. :type keys: Key | str | list of Key | list of str :param config: Optional configuration to use for this request. This must be specified\ as a keyword argument. :type config: dict :raises: :exc:`TransactionFailedError`, if the deletion could not be committed. """ if conf["viur.db.caching"] > 0: if isinstance(keys, datastore_types.Key) or isinstance( keys, basestring): #Just one: memcache.delete(str(keys), namespace=__CacheKeyPrefix__, seconds=__cacheLockTime__) elif isinstance(keys, list): for key in keys: assert isinstance(key, datastore_types.Key) or isinstance( key, basestring) memcache.delete(str(key), namespace=__CacheKeyPrefix__, seconds=__cacheLockTime__) return (datastore.Delete(keys, **kwargs))
def execute(self): self.select.execute() # This is a little bit more inefficient than just doing a keys_only query and # sending it to delete, but I think this is the sacrifice to make for the unique caching layer keys = [] def spawn_query(kind, key): qry = Query(kind) qry["__key__ ="] = key return qry queries = [spawn_query(self.select.db_table, x.key()) for x in self.select.results] if not queries: return for entity in QueryByKeys(self.select.model, queries, []).Run(): keys.append(entity.key()) # Delete constraints if that's enabled if constraints.constraint_checks_enabled(self.select.model): constraints.release(self.select.model, entity) caching.remove_entity_from_cache_by_key(entity.key()) datastore.Delete(keys)
def post(self): kind = self.request.get('kind') entity_key = self.request.get('key') if entity_key: if self.request.get('action') == 'Delete': datastore.Delete(datastore.Key(entity_key)) self.redirect(self.request.get('next')) return entity = datastore.Get(datastore.Key(entity_key)) else: entity = datastore.Entity(kind) args = self.request.arguments() for arg in args: bar = arg.find('|') if bar > 0: data_type_name = arg[:bar] field_name = arg[bar + 1:] form_value = self.request.get(arg) data_type = DataType.get_by_name(data_type_name) if entity and entity.has_key(field_name): old_formatted_value = data_type.format(entity[field_name]) if old_formatted_value == ustr(form_value): continue if len(form_value) > 0: value = data_type.parse(form_value) entity[field_name] = value elif entity.has_key(field_name): del entity[field_name] datastore.Put(entity) self.redirect(self.request.get('next'))
def run(self, context): # This code builds leaf categories for selection with complete names, 3.8k of them. if not isinstance(self.cfg, dict): self.cfg = {} update_file_path = self.cfg.get('file', None) debug_environment = self.cfg.get('debug_environment', False) if not update_file_path: raise orm.TerminateAction() Category = context.models['24'] gets = datastore.Query('24', namespace=None, keys_only=True).Run() keys = list(gets) datastore.Delete(keys) categories = [] put_entities = [] structure = {} with file(update_file_path) as f: for line in f: if not line.startswith('#'): item = line.replace('\n', '') categories.append(item) full_path = item.split(' > ') current_structure = structure for xi, path in enumerate(full_path): if path not in current_structure: current_structure[path] = {} current_structure = current_structure[path] for i, item in enumerate(categories): full_path = item.split(' > ') path_map = structure current = full_path parent = current[:-1] category = {} category['id'] = hashlib.md5(''.join(current)).hexdigest() if parent: category['parent_record'] = Category.build_key( hashlib.md5(''.join(parent)).hexdigest()) else: category['parent_record'] = None category['name'] = ' / '.join(current) category['state'] = ['indexable'] leaf = False for path in full_path: if path in path_map: path_map = path_map[path] if not len(path_map): leaf = True if leaf: category['state'].append( 'visible') # marks the category as leaf category = Category(**category) category._use_rule_engine = False category._use_record_engine = False category._use_memcache = False category._use_cache = False put_entities.append(category) tools.log.debug('Writing %s categories' % len(put_entities)) orm.put_multi(put_entities)
def Dispatch(self, request, outfile, base_env_dict=None): """Handle post dispatch. This dispatcher will handle all uploaded files in the POST request, store the results in the blob-storage, close the upload session and transform the original request in to one where the uploaded files have external bodies. Returns: New AppServerRequest indicating request forward to upload success handler. """ if base_env_dict['REQUEST_METHOD'] != 'POST': outfile.write('Status: 400\n\n') return upload_key = re.match(UPLOAD_URL_PATTERN, request.relative_url).group(1) try: upload_session = datastore.Get(upload_key) except datastore_errors.EntityNotFoundError: upload_session = None if upload_session: success_path = upload_session['success_path'] upload_form = cgi.FieldStorage(fp=request.infile, headers=request.headers, environ=base_env_dict) try: mime_message_string = self.__cgi_handler.GenerateMIMEMessageString( upload_form) datastore.Delete(upload_session) self.current_session = upload_session header_end = mime_message_string.find('\n\n') + 1 content_start = header_end + 1 header_text = mime_message_string[:header_end] content_text = mime_message_string[content_start:] complete_headers = ('%s' 'Content-Length: %d\n' '\n') % (header_text, len(content_text)) return dev_appserver.AppServerRequest( success_path, None, mimetools.Message( cStringIO.StringIO(complete_headers)), cStringIO.StringIO(content_text), force_admin=True) except dev_appserver_upload.InvalidMIMETypeFormatError: outfile.write('Status: 400\n\n') else: logging.error('Could not find session for %s', upload_key) outfile.write('Status: 404\n\n')
def DeleteMultiple(self, node_indexes): """Delete multiple nodes at once. Args: node_indexes: [node index, ...] # where node_index is an integer """ db_nodes = datastore.Delete([ self._RankerNodeKey(node_index) for node_index in set(node_indexes) ])
def delete_all(self, kind, batch_size=1000): from google.appengine.api import datastore count = 0 query = datastore.Query(kind=kind, keys_only=True) results = query.Get(batch_size) while results: datastore.Delete(results) count += len(results) results = query.Get(batch_size) return count
def syncDeletedEntity(self, key): """Delete entity. :param string key: The remote key. """ sync_info = SyncInfo.get_by_key_name(key) datastore.Delete([sync_info.target_key(), sync_info.key()]) return {"status": ENTITY_DELETED}
def execute(self): table = self.table query = datastore.Query(table, keys_only=True) while query.Count(): datastore.Delete(query.Run()) cache.clear() from .caching import clear_context_cache clear_context_cache()
def removeAssociation(self, server_url, handle): """ This method removes the matching association if it's found, and returns whether the association was removed or not. """ query = datastore.Query('Association', {'url =': server_url, 'handle =': handle}) results = query.Get(1) if results: datastore.Delete(results[0].key())
def DeleteUrlBase(self, request, response): """Trivial implementation of ImagesService::DeleteUrlBase. Args: request: ImagesDeleteUrlBaseRequest, contains a blobkey to an image. response: ImagesDeleteUrlBaseResonse - currently unused. """ key = datastore.Key.from_path(BLOB_SERVING_URL_KIND, request.blob_key(), namespace="") datastore.Delete(key)
def tearDown(self): """Clean up.""" query = blobstore.BlobInfo.all() cursor = query.fetch(10) for b in cursor: key = datastore_types.Key.from_path('__BlobInfo__', str(b.key())) datastore.Delete(key) os.unlink(self.datastore_path)
def respond(self): models = io.Engine.get_schema() kinds = ['0', '6', '83', '5', '35', '36', '62', '61', '39', '38', '60', '8', '57', '77', '10', '15', '16', '17', '18', '19', '49', '47'] namespaces = metadata.get_namespaces() indexes = [] keys_to_delete = [] if self.request.get('kinds'): kinds = self.request.get('kinds').split(',') util.log('DELETE KINDS %s' % kinds) ignore = ['15', '16', '17', '18', '19'] @orm.tasklet def wipe(kind): util.log(kind) @orm.tasklet def generator(): model = models.get(kind) if model and not kind.startswith('__'): keys = yield model.query().fetch_async(keys_only=True) keys_to_delete.extend(keys) indexes.append(search.Index(name=kind)) for namespace in namespaces: util.log(namespace) keys = yield model.query(namespace=namespace).fetch_async(keys_only=True) keys_to_delete.extend(keys) indexes.append(search.Index(name=kind, namespace=namespace)) yield generator() if self.request.get('delete'): futures = [] for kind in kinds: if kind not in ignore: futures.append(wipe(kind)) orm.Future.wait_all(futures) if self.request.get('and_system'): futures = [] for kind in kinds: if kind in ignore: futures.append(wipe(kind)) orm.Future.wait_all(futures) if keys_to_delete: datastore.Delete([key.to_old_key() for key in keys_to_delete]) indexes.append(search.Index(name='catalogs')) # empty catalog index! for index in indexes: while True: document_ids = [document.doc_id for document in index.get_range(ids_only=True)] if not document_ids: break try: index.delete(document_ids) except: pass mem.flush_all()
def DeleteUrlBase(self, request, unused_response): """Trivial implementation of an API call. Args: request: ImagesDeleteUrlBaseRequest - Contains a blobkey to an image. unused_response: ImagesDeleteUrlBaseResponse - Unused. """ key = datastore.Key.from_path(BLOB_SERVING_URL_KIND, request.blob_key(), namespace='') datastore.Delete(key)
def deleteUserData(id): # memcache.delete(key="MakerSecret-"+id) # memcache.delete(key="USER-"+id) # dashid=memcache.get(key="User-dash-"+id) # memcache.delete(key = "User-dash-"+id) # memcache.delete(key = "Dash-user-"+str(dashid)) try: key = Key.from_path('UserData', id) entity = datastore.Delete(key) except: logging.debug(id + u"は登録されていません。")
def _Dynamic_DeleteBlob(self, request, response): """Delete a blob by its blob-key. Delete a blob from the blobstore using its blob-key. Deleting blobs that do not exist is a no-op. Args: request: A fully initialized DeleteBlobRequest instance. response: Not used but should be a VoidProto. """ for blob_key in request.blob_key_list(): key = datastore_types.Key.from_path('__BlobInfo__', str(blob_key)) self.__storage.DeleteBlob(key) datastore.Delete(key)
def flush_task_markers(): """ Delete all ShardedTaskMarker objects from the DB. Useful to call in setUp(), as Django doesn't wipe this kind because there's no model for it. """ namespaces = set() namespaces.add(settings.DATABASES['default'].get('NAMESPACE', '')) namespaces.add(settings.DATABASES.get('ns1', {}).get('NAMESPACE', '')) for namespace in namespaces: query = datastore.Query(ShardedTaskMarker.KIND, namespace=namespace, keys_only=True).Run() datastore.Delete([x for x in query])