Exemplo n.º 1
0
    def write(self, data, params=None):
        to_make_partition = True
        if params and 'make_partition' in params:
            to_make_partition = params['make_partition']
            params.pop('make_partition', None)

        if self._underlying_storage.get_storage_type(
        ) == StorageType.PROTO_TABLE_STORAGE:
            file_base_name = 'data.pb'
        else:
            file_base_name = 'data'
        if params and 'base_name' in params:
            file_base_name = params['base_name']
            params.pop('base_name', None)

        while self._reader_status != Status.IDLE:
            self.sys_log("Waiting for reader to finish.")
            time.sleep(TimeSleepObj.ONE_SECOND)

        self._writer_status = Status.RUNNING

        if to_make_partition:
            if not params or 'timezone' not in params or params[
                    'timezone'] == 'PST':
                self.make_new_partition(
                    timestamp=TimezoneUtil.cur_time_in_pst())
            elif params['timezone'] == 'UTC':
                self.make_new_partition(
                    timestamp=TimezoneUtil.cur_time_in_utc())
            elif params['timezone'] == 'EST':
                self.make_new_partition(
                    timestamp=TimezoneUtil.cur_time_in_est())

        self.initialize_from_dir(dir_name=self._file_tree.get_root_name())

        file_name = FileUtil.join_paths_to_file(
            root_dir=self._file_tree.get_leftmost_leaf(),
            base_name=file_base_name)

        if file_name != self._underlying_storage.get_file_name():
            self.sys_log("Sync to the latest file to " + file_name)
            self._underlying_storage.initialize_from_file(file_name=file_name)

        try:
            self._underlying_storage.write(data=data, params=params)
            self._writer_status = Status.IDLE
        except Exception as err:
            self.sys_log("Write to dir [" + self.get_dir_name() +
                         "] got exception: " + str(err) + '.')
            self._logger.error("Write to dir [" + self.get_dir_name() +
                               "] got exception: " + str(err) + '.')
            raise StorageWriteException("Write to dir [" +
                                        self.get_dir_name() +
                                        "] got exception: " + str(err) + '.')
Exemplo n.º 2
0
    def write(self, data, params=None):
        to_make_partition = True
        if params and 'make_partition' in params:
            to_make_partition = params['make_partition']
            params.pop('make_partition', None)

        if self._underlying_storage.get_storage_type(
        ) == StorageType.PROTO_TABLE_STORAGE:
            file_base_name = 'data.pb'
        else:
            file_base_name = 'data'
        if params and 'base_name' in params:
            file_base_name = params['base_name']

        if to_make_partition:
            if not params or 'timezone' not in params or params[
                    'timezone'] == 'PST':
                self.make_new_partition(
                    timestamp=TimezoneUtil.cur_time_in_pst())
            elif params['timezone'] == 'UTC':
                self.make_new_partition(
                    timestamp=TimezoneUtil.cur_time_in_utc())
            elif params['timezone'] == 'EST':
                self.make_new_partition(
                    timestamp=TimezoneUtil.cur_time_in_est())

        file_name = FileUtil.join_paths_to_file(
            root_dir=self._file_tree.get_leftmost_leaf(),
            base_name=file_base_name)

        if file_name != self._underlying_storage.get_file_name():
            self._SYS_LOGGER.info("Sync to the latest file to " + file_name)
            self._underlying_storage.initialize_from_file(file_name=file_name)

        try:
            self._underlying_storage.write(data=data, params=params)
            self.increment_rpc_count_by(
                n=self._underlying_storage.get_rpc_call_count_and_reset())
        except Exception as err:
            self._SYS_LOGGER.error("Write to dir [" + self.get_dir_name() +
                                   "] got exception: " + str(err) + '.')
            self._logger.error("Write to dir [" + self.get_dir_name() +
                               "] got exception: " + str(err) + '.')
            raise StorageWriteException("Write to dir [" +
                                        self.get_dir_name() +
                                        "] got exception: " + str(err) + '.')