def write(self, data, params=None): to_make_partition = True if params and 'make_partition' in params: to_make_partition = params['make_partition'] params.pop('make_partition', None) if self._underlying_storage.get_storage_type( ) == StorageType.PROTO_TABLE_STORAGE: file_base_name = 'data.pb' else: file_base_name = 'data' if params and 'base_name' in params: file_base_name = params['base_name'] params.pop('base_name', None) while self._reader_status != Status.IDLE: self.sys_log("Waiting for reader to finish.") time.sleep(TimeSleepObj.ONE_SECOND) self._writer_status = Status.RUNNING if to_make_partition: if not params or 'timezone' not in params or params[ 'timezone'] == 'PST': self.make_new_partition( timestamp=TimezoneUtil.cur_time_in_pst()) elif params['timezone'] == 'UTC': self.make_new_partition( timestamp=TimezoneUtil.cur_time_in_utc()) elif params['timezone'] == 'EST': self.make_new_partition( timestamp=TimezoneUtil.cur_time_in_est()) self.initialize_from_dir(dir_name=self._file_tree.get_root_name()) file_name = FileUtil.join_paths_to_file( root_dir=self._file_tree.get_leftmost_leaf(), base_name=file_base_name) if file_name != self._underlying_storage.get_file_name(): self.sys_log("Sync to the latest file to " + file_name) self._underlying_storage.initialize_from_file(file_name=file_name) try: self._underlying_storage.write(data=data, params=params) self._writer_status = Status.IDLE except Exception as err: self.sys_log("Write to dir [" + self.get_dir_name() + "] got exception: " + str(err) + '.') self._logger.error("Write to dir [" + self.get_dir_name() + "] got exception: " + str(err) + '.') raise StorageWriteException("Write to dir [" + self.get_dir_name() + "] got exception: " + str(err) + '.')
def write(self, data, params=None): to_make_partition = True if params and 'make_partition' in params: to_make_partition = params['make_partition'] params.pop('make_partition', None) if self._underlying_storage.get_storage_type( ) == StorageType.PROTO_TABLE_STORAGE: file_base_name = 'data.pb' else: file_base_name = 'data' if params and 'base_name' in params: file_base_name = params['base_name'] if to_make_partition: if not params or 'timezone' not in params or params[ 'timezone'] == 'PST': self.make_new_partition( timestamp=TimezoneUtil.cur_time_in_pst()) elif params['timezone'] == 'UTC': self.make_new_partition( timestamp=TimezoneUtil.cur_time_in_utc()) elif params['timezone'] == 'EST': self.make_new_partition( timestamp=TimezoneUtil.cur_time_in_est()) file_name = FileUtil.join_paths_to_file( root_dir=self._file_tree.get_leftmost_leaf(), base_name=file_base_name) if file_name != self._underlying_storage.get_file_name(): self._SYS_LOGGER.info("Sync to the latest file to " + file_name) self._underlying_storage.initialize_from_file(file_name=file_name) try: self._underlying_storage.write(data=data, params=params) self.increment_rpc_count_by( n=self._underlying_storage.get_rpc_call_count_and_reset()) except Exception as err: self._SYS_LOGGER.error("Write to dir [" + self.get_dir_name() + "] got exception: " + str(err) + '.') self._logger.error("Write to dir [" + self.get_dir_name() + "] got exception: " + str(err) + '.') raise StorageWriteException("Write to dir [" + self.get_dir_name() + "] got exception: " + str(err) + '.')