def _get_manifest(self): manifest_etcd_key = self._get_manifest_etcd_key() manifest_data = self._etcd.get_data(manifest_etcd_key) if manifest_data is not None: return text_format.Parse(manifest_data, dj_pb.RawDataManifest()) raise RuntimeError("partition id {} has no manifest".format( self._partition_id))
def _sync_raw_data_manifest(self, partition_id): kvstore_key = partition_manifest_kvstore_key(self._data_source_name(), partition_id) data = self._kvstore.get_data(kvstore_key) assert data is not None, "raw data manifest of partition "\ "{} must be existed".format(partition_id) return text_format.Parse(data, dj_pb.RawDataManifest())
def _get_manifest(self, partition_id): manifest_etcd_key = common.partition_manifest_etcd_key( self._data_source.data_source_meta.name, partition_id) manifest_data = self._etcd.get_data(manifest_etcd_key) if manifest_data is not None: return text_format.Parse(manifest_data, dj_pb.RawDataManifest()) return None
def _sync_raw_data_manifest(self): etcd_key = common.partition_manifest_etcd_key( self._data_source.data_source_meta.name, self._partition_id ) data = self._etcd.get_data(etcd_key) assert data is not None, "manifest must be existed" return text_format.Parse(data, dj_pb.RawDataManifest())
def _sync_raw_data_manifest(self): kvstore_key = common.partition_manifest_kvstore_key( self._data_source.data_source_meta.name, self._partition_id) data = self._kvstore.get_data(kvstore_key) assert data is not None, "manifest must be existed" return text_format.Parse(data, dj_pb.RawDataManifest(), allow_unknown_field=True)
def _init_manifest(self, partition_id): mainfest = self._get_manifest(partition_id) if mainfest is None: manifest = dj_pb.RawDataManifest() manifest.state = dj_pb.RawDataState.UnAllocated manifest.partition_id = partition_id manifest.allocated_rank_id = -1 self._update_manifest(partition_id, manifest) self._local_manifest[partition_id] = mainfest
def _get_manifest(self, partition_id): manifest_kvstore_key = common.partition_manifest_kvstore_key( self._data_source.data_source_meta.name, partition_id ) manifest_data = self._kvstore.get_data(manifest_kvstore_key) if manifest_data is not None: return text_format.Parse(manifest_data, dj_pb.RawDataManifest(), allow_unknown_field=True) return None
def _sync_manifest(self, partition_id): assert partition_id in self._local_manifest, \ "partition {} should in local manifest".format(partition_id) if self._local_manifest[partition_id] is None: manifest = self._get_manifest(partition_id) if manifest is None: manifest = dj_pb.RawDataManifest() manifest.sync_example_id_rep.rank_id = -1 manifest.join_example_rep.rank_id = -1 manifest.partition_id = partition_id self._update_manifest(manifest) self._process_next_process_index(partition_id, manifest) return self._local_manifest[partition_id]
def _get_manifest(self, partition_id): manifest_etcd_key = self._get_manifest_etcd_key(partition_id) manifest_data = self._etcd.get_data(manifest_etcd_key) if manifest_data is not None: return text_format.Parse(manifest_data, dj_pb.RawDataManifest()) return None