def _sync_raw_data_meta(self, process_index): kvstore_key = common.raw_data_meta_kvstore_key( self._data_source.data_source_meta.name, self._partition_id, process_index) data = self._kvstore.get_data(kvstore_key) if data is not None: return text_format.Parse(data, dj_pb.RawDataMeta()) return None
def _new_index_meta(self, process_index, start_index): if self._manifest.next_process_index <= process_index: return None raw_data_meta = None if process_index < len(self._all_metas): assert process_index == self._all_metas[process_index][0], \ "process index should equal {} != {}".format( process_index, self._all_metas[process_index][0] ) raw_data_meta = self._all_metas[process_index][1] else: assert process_index == len(self._all_metas), \ "the process index should be the next all metas "\ "{}(process_index) != {}(size of all_metas)".format( process_index, len(self._all_metas) ) raw_data_meta = self._sync_raw_data_meta(process_index) if raw_data_meta is None: logging.fatal("the raw data of partition %d index with "\ "%d must in kvstore", self._partition_id, process_index) traceback.print_stack() os._exit(-1) # pylint: disable=protected-access self._all_metas.append((process_index, raw_data_meta)) if raw_data_meta.start_index == -1: new_meta = dj_pb.RawDataMeta() new_meta.MergeFrom(raw_data_meta) new_meta.start_index = start_index odata = text_format.MessageToString(raw_data_meta) ndata = text_format.MessageToString(new_meta) kvstore_key = common.raw_data_meta_kvstore_key( self._data_source.data_source_meta.name, self._partition_id, process_index) if not self._kvstore.cas(kvstore_key, odata, ndata): raw_data_meta = self._sync_raw_data_meta(process_index) assert raw_data_meta is not None, \ "the raw data meta of process index {} "\ "must not None".format(process_index) if raw_data_meta.start_index != start_index: logging.fatal("raw data of partition %d index with "\ "%d must start with %d", self._partition_id, process_index, start_index) traceback.print_stack() os._exit(-1) # pylint: disable=protected-access return visitor.IndexMeta(process_index, start_index, raw_data_meta.file_path)
def _store_raw_data_metas(self, partition_id, new_raw_data_metas): if len(new_raw_data_metas) > 0: manifest = self._sync_manifest(partition_id) self._local_manifest[partition_id] = None process_index = manifest.next_process_index for raw_date_meta in new_raw_data_metas: kvstore_key = common.raw_data_meta_kvstore_key( self._data_source.data_source_meta.name, partition_id, process_index) raw_date_meta.start_index = -1 data = text_format.MessageToString(raw_date_meta) self._kvstore.set_data(kvstore_key, data) self._existed_fpath[raw_date_meta.file_path] = \ (partition_id, process_index) self._update_raw_data_latest_timestamp(partition_id, raw_date_meta.timestamp) process_index += 1 if manifest.next_process_index != process_index: manifest.next_process_index = process_index self._update_manifest(manifest) else: self._local_manifest[partition_id] = manifest
def _process_next_process_index(self, partition_id, manifest): assert manifest is not None and manifest.partition_id == partition_id next_process_index = manifest.next_process_index while True: meta_kvstore_key = \ common.raw_data_meta_kvstore_key( self._data_source.data_source_meta.name, partition_id, next_process_index ) data = self._kvstore.get_data(meta_kvstore_key) if data is None: break meta = text_format.Parse(data, dj_pb.RawDataMeta()) self._existed_fpath[meta.file_path] = \ (partition_id, next_process_index) self._update_raw_data_latest_timestamp(partition_id, meta.timestamp) next_process_index += 1 if next_process_index != manifest.next_process_index: manifest.next_process_index = next_process_index self._update_manifest(manifest) else: self._local_manifest[partition_id] = manifest