def delete(self): """Deletes a RecordIO. Modifying RecordIOs or applying queued writes may result in errors during deletions. """ db.delete(RecordIOShard.get_all_query(self.name, keys_only=True))
def create(self, compressed=True, pre_split=[]): """Creates a RecordIO in datastore. If the RecordIO exists, nothing happens :param compressed: Boolean if the data in the RecordIO should be gzipped. :param pre_split: An optional list of keys to that should be used to pre-split the internal data shards. This is only makes sense if you are going to write a lot of data and you already know the key range of the data and roughly how many entries fit into one shard. :return: True, if the RecordIO didn't exist before. """ self.db_search += 1 if RecordIOShard.get_all_query(self.name, keys_only=True).get() == None: pre_split.sort() self.db_put += 1 split = [None] + [(x, ) for x in pre_split] + [None] split = [(split[i], split[i + 1]) for i in xrange(len(split) - 1)] for lo, hi in split: index = None if lo == None: index = True RecordIOShard.get_or_insert(RecordIOShard.key_name(self.name, lo=lo, hi=hi), compressed=compressed, index=index) return True return False
def create(self, compressed=True, pre_split=[]): """Creates a RecordIO in datastore. If the RecordIO exists, nothing happens :param compressed: Boolean if the data in the RecordIO should be gzipped. :param pre_split: An optional list of keys to that should be used to pre-split the internal data shards. This is only makes sense if you are going to write a lot of data and you already know the key range of the data and roughly how many entries fit into one shard. :return: True, if the RecordIO didn't exist before. """ self.db_search += 1 if RecordIOShard.get_all_query(self.name, keys_only=True).get() == None: pre_split.sort() self.db_put += 1 split = [None] + [(x,) for x in pre_split] + [None] split = [(split[i], split[i+1]) for i in xrange(len(split) - 1)] for lo, hi in split: index = None if lo == None: index = True RecordIOShard.get_or_insert(RecordIOShard.key_name(self.name, lo=lo, hi=hi), compressed=compressed, index=index) return True return False
def testGetAllQuery(self): RecordIOShard.create("test", hi=("a", "")).commit() RecordIOShard.create("test", lo=("a", ""), hi=("b", "")).commit() RecordIOShard.create("test", lo=("b", "")).commit() self.assertEqual( [(None, ("a", 0, 1, 0)), (('a', 0, 1, 0), ('b', 0, 1, 0)), (('b', 0, 1, 0), None)], [RecordIOShard.lo_hi_from_key(x.name()) for x in RecordIOShard.get_all_query("test", keys_only=True)])