def delete_all(self):
while self._reader_status != Status.IDLE:
self.sys_log("Waiting for reader to finish.")
time.sleep(TimeSleepObj.ONE_SECOND)
while self._writer_status != Status.IDLE:
self.sys_log("Waiting for writer to finish.")
time.sleep(TimeSleepObj.ONE_SECOND)
self._deleter_status = Status.RUNNING
all_keys = list(dict(self._table_message.data).keys())
for key in all_keys:
del self._table_message.data[key]
try:
self._table_message.updated_time = str(TimezoneUtil.cur_time_in_pst())
with FileLockTool(self._file_name, read_mode=False):
FileUtil.write_proto_to_file(
proto=self._table_message,
file_name=self._file_name
)
self._deleter_status = Status.IDLE
except Exception as err:
self.sys_log("Delete all of file [" + self.get_file_name() + "] got exception: " + str(err) + '.')
self._logger.error("Delete all of file [" + self.get_file_name() + "] got exception: " + str(err) + '.')
raise StorageDeleteException("Delete all of file [" + self.get_file_name() +
"] got exception: " + str(err) + '.')
def write(self, data, params=None):
if not params:
params = {}
if 'overwrite' not in params:
params['overwrite'] = True
assert isinstance(data, dict)
try:
for key, val in data.items():
if not params['overwrite'] and key in self._table_message.data:
continue
any_message = ProtoUtil.message_to_any(message=val)
self._table_message.data[key].CopyFrom(any_message)
if len(self._table_message.data) > 1000:
self._SYS_LOGGER.warning(
"Warning: the table content is too large, considering using Partitioner "
"combined with proto table.")
self._table_message.updated_time = str(
TimezoneUtil.cur_time_in_pst())
self.increment_rpc_count_by(n=1)
FileUtil.write_proto_to_file(proto=self._table_message,
file_name=self._file_name)
except Exception as err:
self._SYS_LOGGER.error("Write to file [" + self.get_file_name() +
"] got exception: " + str(err) + '.')
self._logger.error("Write to file [" + self.get_file_name() +
"] got exception: " + str(err) + '.')
raise StorageWriteException("Write to file [" +
self.get_file_name() +
"] got exception: " + str(err) + '.')
def get_container_snapshot(self, send_backend=True):
if not self._is_initialized:
self._logger.error("Warning: taking snapshot when the container [" + self.get_container_name() +
"] is not initialized.")
self.sys_log("Warning: taking snapshot when the container [" + self.get_container_name() +
"] is not initialized.")
snapshot = ContainerSnapshot()
snapshot.container_name = self._container_name
snapshot.is_initialized = self._is_initialized
snapshot.status = self._status
snapshot.class_name = self.get_full_class_name()
snapshot.mode = self._mode
snapshot.data_model = self.DATA_MODEL
snapshot.log_dir = self._logger.get_log_dir()
for key, val in self._counter.items():
snapshot.counters[key] = val
if self._start_time:
snapshot.start_time = str(self._start_time)
if self._end_time:
snapshot.end_time = str(self._end_time)
for op_name, op in self._node_name_to_node_dict.items():
if 'Dummy' in op.get_class_name():
continue
op_output_file = FileUtil.join_paths_to_file(
root_dir=FileUtil.join_paths_to_dir(FileUtil.dir_name(self._snapshot_file_folder), 'operators'),
base_name='SNAPSHOT_' + str(TimezoneUtil.cur_time_in_pst()) + '_' + op_name + '.pb'
)
snapshot.operator_snapshot_map[op_name].CopyFrom(op.get_operator_snapshot(output_file=op_output_file))
self.sys_log("Snapshot saved to folder [" + self._snapshot_file_folder + '].')
self._logger.info("Snapshot saved to folder [" + self._snapshot_file_folder + '].')
output_file_name = FileUtil.join_paths_to_file(
root_dir=FileUtil.dir_name(self._snapshot_file_folder),
base_name='SNAPSHOT_' + str(TimezoneUtil.cur_time_in_pst()) + '_' + self._container_name + '.pb'
)
with FileLockTool(output_file_name, read_mode=False):
FileUtil.write_proto_to_file(
proto=snapshot,
file_name=output_file_name
)
if self._backend and send_backend:
try:
self._backend.send_to_backend(snapshot=snapshot)
except Exception as err:
self._logger.error("Sending backend failed with error " + str(err) + '.')
return snapshot
def delete_multiple(self, keys):
for key in keys:
if key in self._table_message.data:
del self._table_message.data[key]
try:
self._table_message.updated_time = str(
TimezoneUtil.cur_time_in_pst())
self.increment_rpc_count_by(n=1)
FileUtil.write_proto_to_file(proto=self._table_message,
file_name=self._file_name)
except Exception as err:
self._SYS_LOGGER.error("Delete file [" + self.get_file_name() +
"] got exception: " + str(err))
self._logger.error("Delete file [" + self.get_file_name() +
"] got exception: " + str(err))
raise StorageDeleteException("Delete file [" +
self.get_file_name() +
"] got exception: " + str(err))
def delete_all(self):
all_keys = list(dict(self._table_message.data).keys())
for key in all_keys:
del self._table_message.data[key]
try:
self._table_message.updated_time = str(
TimezoneUtil.cur_time_in_pst())
self.increment_rpc_count_by(n=1)
FileUtil.write_proto_to_file(proto=self._table_message,
file_name=self._file_name)
except Exception as err:
self._SYS_LOGGER.error("Delete all of file [" +
self.get_file_name() + "] got exception: " +
str(err) + '.')
self._logger.error("Delete all of file [" + self.get_file_name() +
"] got exception: " + str(err) + '.')
raise StorageDeleteException("Delete all of file [" +
self.get_file_name() +
"] got exception: " + str(err) + '.')
def get_operator_snapshot(self, output_file=None):
snapshot = OperatorSnapshot()
snapshot.operator_name = self.get_node_name()
snapshot.data_model = self.get_data_model()
snapshot.status = self.get_status()
snapshot.node_snapshot.CopyFrom(self.get_node_snapshot())
snapshot.class_name = self.get_full_class_name()
if self._start_time:
snapshot.start_time = str(self._start_time)
if self.get_status() == Status.SUCCEEDED and self._end_time:
snapshot.end_time = str(self._end_time)
if self._persistent:
assert self.CONTENT_MESSAGE_TYPE is not None
snapshot.content.CopyFrom(ProtoUtil.message_to_any(message=self._content))
if output_file and self._config['save_snapshot'] and 'Dummy' not in self.get_class_name():
self.sys_log("Saved to file " + output_file + '.')
with FileLockTool(output_file, read_mode=False):
FileUtil.write_proto_to_file(
proto=snapshot,
file_name=output_file
)
return snapshot
def get_operator_snapshot(self, output_file=None):
snapshot = OperatorSnapshot()
snapshot.operator_name = self.get_node_name()
snapshot.data_model = self.get_data_model()
snapshot.status = self.get_status()
snapshot.node_snapshot.CopyFrom(self.get_node_snapshot())
snapshot.class_name = self.get_full_class_name()
snapshot.log_file = FileUtil.convert_local_to_cell_path(glogging.get_logger_file(self._logger))
if self._start_time:
snapshot.start_time = str(self._start_time)
if self.get_status() == Status.SUCCEEDED and self._end_time:
snapshot.end_time = str(self._end_time)
if self._persistent:
snapshot.content.CopyFrom(self.content_serializer(self._content))
if output_file and self._config['save_snapshot'] and 'Dummy' not in self.get_class_name():
self._SYS_LOGGER.info("Saved to file " + output_file + '.')
FileUtil.write_proto_to_file(
proto=snapshot,
file_name=output_file
)
return snapshot
def write(self, data, params=None):
if not params:
params = {}
if 'overwrite' not in params:
params['overwrite'] = True
while self._reader_status != Status.IDLE:
self.sys_log("Waiting for reader to finish.")
time.sleep(TimeSleepObj.ONE_SECOND)
while self._deleter_status != Status.IDLE:
self.sys_log("Waiting for deleter to finish.")
time.sleep(TimeSleepObj.ONE_SECOND)
self._writer_status = Status.RUNNING
assert isinstance(data, dict)
try:
for key, val in data.items():
if not params['overwrite'] and key in self._table_message.data:
continue
any_message = ProtoUtil.message_to_any(message=val)
self._table_message.data[key].CopyFrom(any_message)
if len(self._table_message.data) > 1000:
self.sys_log("Warning: the table content is too large, considering using Partitioner "
"combined with proto table.")
self._table_message.updated_time = str(TimezoneUtil.cur_time_in_pst())
with FileLockTool(self._file_name, read_mode=False):
FileUtil.write_proto_to_file(
proto=self._table_message,
file_name=self._file_name
)
self._writer_status = Status.IDLE
except Exception as err:
self.sys_log("Write to file [" + self.get_file_name() + "] got exception: " + str(err) + '.')
self._logger.error("Write to file [" + self.get_file_name() + "] got exception: " + str(err) + '.')
raise StorageWriteException("Write to file [" + self.get_file_name() + "] got exception: " + str(err) + '.')
def write(self, data, params=None):
if not params:
params = {}
if 'overwrite' not in params:
params['overwrite'] = True
assert isinstance(data, dict)
exising_shard_to_data_map = defaultdict(dict)
new_data = {}
for key, val in data.items():
if key in self._index_map.index_map:
exising_shard_to_data_map[self._index_map.index_map[key]][key] = val
else:
new_data[key] = val
try:
for shard, existing_data in exising_shard_to_data_map.items():
related_proto_file = self._shard_to_file(shard=shard)
proto_table = ProtoTableStorage()
proto_table.initialize_from_file(file_name=related_proto_file)
proto_table.write(data=existing_data, params=params)
self.increment_rpc_count_by(n=proto_table.get_rpc_call_count_and_reset())
if new_data:
all_new_keys = list(new_data.keys())
latest_shard = self.get_latest_shard()
latest_proto_file = self._shard_to_file(shard=latest_shard)
proto_table = ProtoTableStorage()
proto_table.initialize_from_file(file_name=latest_proto_file)
latest_proto_table_size = proto_table.get_num_entries()
proto_table.write(
data={key: new_data[key] for key in all_new_keys[:self._size_per_shard - latest_proto_table_size]},
params=params
)
self.increment_rpc_count_by(n=proto_table.get_rpc_call_count_and_reset())
for key in all_new_keys[:self._size_per_shard - latest_proto_table_size]:
self._index_map.index_map[key] = latest_shard
if len(all_new_keys) > self._size_per_shard - latest_proto_table_size:
remaining_new_keys = all_new_keys[self._size_per_shard - latest_proto_table_size:]
start_index = 0
while start_index < len(remaining_new_keys):
data_for_new_shard = remaining_new_keys[start_index:start_index + self._size_per_shard]
latest_shard += 1
self._index_map.cur_shard += 1
proto_file = self._shard_to_file(shard=latest_shard)
self._logger.info("Write to new file with name [" + proto_file + '] and shard [' +
str(latest_shard) + '].')
proto_table = ProtoTableStorage()
proto_table.initialize_from_file(file_name=proto_file)
proto_table.write(
data={key: new_data[key] for key in data_for_new_shard},
params=params
)
self.increment_rpc_count_by(n=proto_table.get_rpc_call_count_and_reset())
for key in data_for_new_shard:
self._index_map.index_map[key] = latest_shard
start_index += self._size_per_shard
self._logger.info("Writing the index map to [" + self._index_map_file + '].')
self.increment_rpc_count_by(n=1)
FileUtil.write_proto_to_file(
proto=self._index_map,
file_name=self._index_map_file
)
except Exception as err:
self._SYS_LOGGER.error("Write to dir [" + self.get_dir_name() + "] got exception: " + str(err) + '.')
self._logger.error("Write to dir [" + self.get_dir_name() + "] got exception: " + str(err) + '.')
raise StorageWriteException("Write to dir [" + self.get_dir_name() + "] got exception: " + str(err) + '.')
def main():
yaml_path = "example/frontend_example/frontend_config.yaml"
proto_path = "example/frontend_example/frontend_config.pb"
config = CommonUtil.make_frontend_config(yaml_path=yaml_path)
print(config)
FileUtil.write_proto_to_file(proto=config, file_name=proto_path)
def get_container_snapshot(self, send_backend=True):
if not self._is_initialized:
self._logger.error(
"Warning: taking snapshot when the container [" +
self.get_container_name() + "] is not initialized.")
self._SYS_LOGGER.error(
"Warning: taking snapshot when the container [" +
self.get_container_name() + "] is not initialized.")
snapshot = ContainerSnapshot()
snapshot.container_name = self._container_name
snapshot.is_initialized = self._is_initialized
snapshot.status = self._status
snapshot.class_name = self.get_full_class_name()
snapshot.mode = self._mode
snapshot.data_model = self.DATA_MODEL
snapshot.log_file = FileUtil.convert_local_to_cell_path(
glogging.get_logger_file(self._logger))
snapshot.run_cell = EnvUtil.get_other_env_variable(
var='GALAXY_fs_cell', fallback_value='')
snapshot.snapshot_cell = FileUtil.get_cell_from_path(
FileUtil.convert_local_to_cell_path(self._snapshot_file_folder))
for key, val in self._counter.items():
snapshot.counters[key] = val
if self._start_time:
snapshot.start_time = str(self._start_time)
if self._end_time:
snapshot.end_time = str(self._end_time)
for op_name, op in self._node_name_to_node_dict.items():
if 'Dummy' in op.get_class_name():
continue
op_output_file = FileUtil.join_paths_to_file(
root_dir=FileUtil.join_paths_to_dir(
FileUtil.dir_name(self._snapshot_file_folder),
'operators'),
base_name=op_name + '_SNAPSHOT_' +
str(TimezoneUtil.cur_time_in_pst()) + '.pb')
snapshot.operator_snapshot_map[op_name].CopyFrom(
op.get_operator_snapshot(output_file=op_output_file))
self._SYS_LOGGER.info(
"Snapshot saved to folder [" +
FileUtil.convert_local_to_cell_path(self._snapshot_file_folder) +
'].')
self._logger.info(
"Snapshot saved to folder [" +
FileUtil.convert_local_to_cell_path(self._snapshot_file_folder) +
'].')
output_file_name = FileUtil.join_paths_to_file(
root_dir=FileUtil.join_paths_to_dir(
FileUtil.dir_name(self._snapshot_file_folder), 'containers'),
base_name=self._container_name + '_SNAPSHOT_' +
str(TimezoneUtil.cur_time_in_pst()) + '.pb')
FileUtil.write_proto_to_file(proto=snapshot,
file_name=output_file_name)
if self._backend and send_backend:
try:
self._backend.send_to_backend(snapshot=snapshot)
except Exception as err:
self._logger.error("Sending backend failed with error " +
str(err) + '.')
return snapshot
