def _sync_raw_data_manifest(self, partition_id): kvstore_key = partition_manifest_kvstore_key(self._data_source_name(), partition_id) data = self._kvstore.get_data(kvstore_key) assert data is not None, "raw data manifest of partition "\ "{} must be existed".format(partition_id) return text_format.Parse(data, dj_pb.RawDataManifest())
def _preload_raw_data_meta(self): manifest_kvstore_key = common.partition_manifest_kvstore_key( self._data_source.data_source_meta.name, self._partition_id) all_metas = [] index_metas = [] for key, val in self._kvstore.get_prefix_kvs(manifest_kvstore_key, True): bkey = os.path.basename(key) if not bkey.decode().startswith(common.RawDataMetaPrefix): continue index = int(bkey[len(common.RawDataMetaPrefix):]) meta = text_format.Parse(val, dj_pb.RawDataMeta()) all_metas.append((index, meta)) if meta.start_index != -1: index_meta = visitor.IndexMeta(index, meta.start_index, meta.file_path) index_metas.append(index_meta) all_metas = sorted(all_metas, key=lambda meta: meta[0]) for process_index, meta in enumerate(all_metas): if process_index != meta[0]: logging.fatal("process_index mismatch with index %d != %d "\ "for file path %s", process_index, meta[0], meta[1].file_path) traceback.print_stack() os._exit(-1) # pylint: disable=protected-access return all_metas, index_metas
def _get_manifest(self, partition_id): manifest_kvstore_key = common.partition_manifest_kvstore_key( self._data_source.data_source_meta.name, partition_id) manifest_data = self._kvstore.get_data(manifest_kvstore_key) if manifest_data is not None: return text_format.Parse(manifest_data, dj_pb.RawDataManifest()) return None
def _sync_raw_data_manifest(self): kvstore_key = common.partition_manifest_kvstore_key( self._data_source.data_source_meta.name, self._partition_id) data = self._kvstore.get_data(kvstore_key) assert data is not None, "manifest must be existed" return text_format.Parse(data, dj_pb.RawDataManifest(), allow_unknown_field=True)
def _update_manifest(self, manifest): partition_id = manifest.partition_id manifest_kvstore_key = common.partition_manifest_kvstore_key( self._data_source.data_source_meta.name, partition_id) self._local_manifest[partition_id] = None self._kvstore.set_data(manifest_kvstore_key, text_format.MessageToString(manifest)) self._local_manifest[partition_id] = manifest