def initialize_from_file(self, file_name):
if '.pb' not in file_name:
self._SYS_LOGGER.warning(
"Please use .pb extension for proto files.")
self._file_name = FileUtil.normalize_file_name(file_name=file_name)
self._table_message = FileUtil.read_proto_from_file(
proto_type=ProtoTable, file_name=self._file_name)
self.increment_rpc_count_by(n=1)
if self._table_message is None:
self._table_message = ProtoTable()
if not self._table_message.table_path:
self._table_message.table_path = self._file_name
if not self._table_message.table_name:
self._table_message.table_name = FileUtil.base_name(
file_name=file_name)
if not self._table_message.created_time:
self._table_message.created_time = str(
TimezoneUtil.cur_time_in_pst())
def initialize_from_dir(self, dir_name):
self._dir_name = dir_name
self._index_map_file = FileUtil.join_paths_to_file(
root_dir=dir_name,
base_name='index_map.pb'
)
self._index_map_file = FileUtil.normalize_file_name(file_name=self._index_map_file)
self.increment_rpc_count_by(n=1)
self._index_map = FileUtil.read_proto_from_file(
proto_type=ProtoTableIndexMap,
file_name=self._index_map_file
)
if self._index_map is None:
self._index_map = ProtoTableIndexMap()
self._index_map.cur_shard = 0
assert self._size_per_shard > 0
self._index_map.size_per_shard = self._size_per_shard
else:
if self._size_per_shard and self._index_map.size_per_shard != self._size_per_shard:
self._logger.error("Please use the correct size per shard of [" + str(self._size_per_shard) + '].')
self._size_per_shard = self._index_map.size_per_shard
self._logger.info("Using size per shard of [" + str(self._size_per_shard) + '].')
def test_normalize_path(self):
dir_name = 'test/foo//'
self.assertEqual(FileUtil.normalize_dir_name(dir_name=dir_name), 'test/foo/')
file_name = 'test/foo.txt'
self.assertEqual(FileUtil.normalize_file_name(file_name=file_name), 'test/foo.txt')
def initialize_from_file(self, file_name):
self._SYS_LOGGER.info("Initialize from file " + file_name + '.')
self._logger.info("Initialize from file " + file_name + '.')
self._file_name = FileUtil.normalize_file_name(file_name=file_name)
self._last_read_line = 0
