def all_names(): """Returns the names of all existing RecordIOs. :return: list of RecordIO names """ for x in RecordIOShard.all(keys_only=True).filter("index =", True): yield RecordIOShard.get_name(x.name())
def all_names(): """Returns the names of all existing RecordIOs. :return: list of RecordIO names """ for x in RecordIOShard.all(keys_only=True).filter("index =", True): yield RecordIOShard.get_name(x.name())
def commit_batch(self, tag, batch): """Applies a batch of values to a RecordIO and deletes the taskqueue task, :param tag: The current tag we are working on :param batch: A list of (tasqueue_task, key_value_list) :return: True on success """ if batch: done_tasks = [] count = 0 writer = RecordIOWriter(RecordIOShard.get_name(tag)) for done_task, key_values in batch: done_tasks.append(done_task) for entry in key_values: writer.insert_entry_(entry) count += 1 try: writer.commit_sync(retries=1) try: self.pull.delete_tasks(done_tasks) except taskqueue.BadTaskStateError: for task in done_tasks: if task.was_deleted: continue try: self.pull.delete_tasks(task) except taskqueue.BadTaskStateError: logging.debug( "RecordIO Failed to free task %s on %s" % task.name, tag) logging.debug("RecordIO wrote %d entries to %s" % (count, writer.name)) except RecordIOWriterNotCompletedError: logging.debug("RecordIO not completed on: %s" % tag) for task in done_tasks: self.pull.modify_task_lease(task, 0) return False return True
def commit_batch(self, tag, batch): """Applies a batch of values to a RecordIO and deletes the taskqueue task, :param tag: The current tag we are working on :param batch: A list of (tasqueue_task, key_value_list) :return: True on success """ if batch: done_tasks = [] count = 0 writer = RecordIOWriter(RecordIOShard.get_name(tag)) for done_task, key_values in batch: done_tasks.append(done_task) for entry in key_values: writer.insert_entry_(entry) count += 1 try: writer.commit_sync(retries=1) try: self.pull.delete_tasks(done_tasks) except taskqueue.BadTaskStateError: for task in done_tasks: if task.was_deleted: continue try: self.pull.delete_tasks(task) except taskqueue.BadTaskStateError: logging.debug("RecordIO Failed to free task %s on %s" % task.name, tag) logging.debug("RecordIO wrote %d entries to %s" % (count, writer.name)) except RecordIOWriterNotCompletedError: logging.debug("RecordIO not completed on: %s" % tag) for task in done_tasks: self.pull.modify_task_lease(task, 0) return False return True