def _sync_raw_data_manifest(self, partition_id):
kvstore_key = partition_manifest_kvstore_key(self._data_source_name(),
partition_id)
data = self._kvstore.get_data(kvstore_key)
assert data is not None, "raw data manifest of partition "\
"{} must be existed".format(partition_id)
return text_format.Parse(data, dj_pb.RawDataManifest())
def _preload_raw_data_meta(self):
manifest_kvstore_key = common.partition_manifest_kvstore_key(
self._data_source.data_source_meta.name, self._partition_id)
all_metas = []
index_metas = []
for key, val in self._kvstore.get_prefix_kvs(manifest_kvstore_key,
True):
bkey = os.path.basename(key)
if not bkey.decode().startswith(common.RawDataMetaPrefix):
continue
index = int(bkey[len(common.RawDataMetaPrefix):])
meta = text_format.Parse(val, dj_pb.RawDataMeta())
all_metas.append((index, meta))
if meta.start_index != -1:
index_meta = visitor.IndexMeta(index, meta.start_index,
meta.file_path)
index_metas.append(index_meta)
all_metas = sorted(all_metas, key=lambda meta: meta[0])
for process_index, meta in enumerate(all_metas):
if process_index != meta[0]:
logging.fatal("process_index mismatch with index %d != %d "\
"for file path %s", process_index, meta[0],
meta[1].file_path)
traceback.print_stack()
os._exit(-1) # pylint: disable=protected-access
return all_metas, index_metas
def _get_manifest(self, partition_id):
manifest_kvstore_key = common.partition_manifest_kvstore_key(
self._data_source.data_source_meta.name, partition_id)
manifest_data = self._kvstore.get_data(manifest_kvstore_key)
if manifest_data is not None:
return text_format.Parse(manifest_data, dj_pb.RawDataManifest())
return None
def _sync_raw_data_manifest(self):
kvstore_key = common.partition_manifest_kvstore_key(
self._data_source.data_source_meta.name, self._partition_id)
data = self._kvstore.get_data(kvstore_key)
assert data is not None, "manifest must be existed"
return text_format.Parse(data,
dj_pb.RawDataManifest(),
allow_unknown_field=True)
def _update_manifest(self, manifest):
partition_id = manifest.partition_id
manifest_kvstore_key = common.partition_manifest_kvstore_key(
self._data_source.data_source_meta.name, partition_id)
self._local_manifest[partition_id] = None
self._kvstore.set_data(manifest_kvstore_key,
text_format.MessageToString(manifest))
self._local_manifest[partition_id] = manifest