def _get_manifest(self):
manifest_etcd_key = self._get_manifest_etcd_key()
manifest_data = self._etcd.get_data(manifest_etcd_key)
if manifest_data is not None:
return text_format.Parse(manifest_data, dj_pb.RawDataManifest())
raise RuntimeError("partition id {} has no manifest".format(
self._partition_id))
def _sync_raw_data_manifest(self, partition_id):
kvstore_key = partition_manifest_kvstore_key(self._data_source_name(),
partition_id)
data = self._kvstore.get_data(kvstore_key)
assert data is not None, "raw data manifest of partition "\
"{} must be existed".format(partition_id)
return text_format.Parse(data, dj_pb.RawDataManifest())
def _get_manifest(self, partition_id):
manifest_etcd_key = common.partition_manifest_etcd_key(
self._data_source.data_source_meta.name, partition_id)
manifest_data = self._etcd.get_data(manifest_etcd_key)
if manifest_data is not None:
return text_format.Parse(manifest_data, dj_pb.RawDataManifest())
return None
def _sync_raw_data_manifest(self):
etcd_key = common.partition_manifest_etcd_key(
self._data_source.data_source_meta.name,
self._partition_id
)
data = self._etcd.get_data(etcd_key)
assert data is not None, "manifest must be existed"
return text_format.Parse(data, dj_pb.RawDataManifest())
def _sync_raw_data_manifest(self):
kvstore_key = common.partition_manifest_kvstore_key(
self._data_source.data_source_meta.name, self._partition_id)
data = self._kvstore.get_data(kvstore_key)
assert data is not None, "manifest must be existed"
return text_format.Parse(data,
dj_pb.RawDataManifest(),
allow_unknown_field=True)
def _init_manifest(self, partition_id):
mainfest = self._get_manifest(partition_id)
if mainfest is None:
manifest = dj_pb.RawDataManifest()
manifest.state = dj_pb.RawDataState.UnAllocated
manifest.partition_id = partition_id
manifest.allocated_rank_id = -1
self._update_manifest(partition_id, manifest)
self._local_manifest[partition_id] = mainfest
def _get_manifest(self, partition_id):
manifest_kvstore_key = common.partition_manifest_kvstore_key(
self._data_source.data_source_meta.name,
partition_id
)
manifest_data = self._kvstore.get_data(manifest_kvstore_key)
if manifest_data is not None:
return text_format.Parse(manifest_data, dj_pb.RawDataManifest(),
allow_unknown_field=True)
return None
def _sync_manifest(self, partition_id):
assert partition_id in self._local_manifest, \
"partition {} should in local manifest".format(partition_id)
if self._local_manifest[partition_id] is None:
manifest = self._get_manifest(partition_id)
if manifest is None:
manifest = dj_pb.RawDataManifest()
manifest.sync_example_id_rep.rank_id = -1
manifest.join_example_rep.rank_id = -1
manifest.partition_id = partition_id
self._update_manifest(manifest)
self._process_next_process_index(partition_id, manifest)
return self._local_manifest[partition_id]
def _get_manifest(self, partition_id):
manifest_etcd_key = self._get_manifest_etcd_key(partition_id)
manifest_data = self._etcd.get_data(manifest_etcd_key)
if manifest_data is not None:
return text_format.Parse(manifest_data, dj_pb.RawDataManifest())
return None
