def _reinitialize_underlying_storage(self, file_base_name): file_name = FileUtil.join_paths_to_file(root_dir=self.get_latest_dir(), base_name=file_base_name) if not FileUtil.does_file_exist(file_name): self.sys_log("The file to read does not exist.") return self._underlying_storage.initialize_from_file(file_name=file_name)
def _reinitialize_underlying_storage(self, file_base_name): file_name = FileUtil.join_paths_to_file( root_dir=self._get_latest_dir_internal(), base_name=file_base_name) self.increment_rpc_count_by(n=1) if not FileUtil.does_file_exist(file_name): self._SYS_LOGGER.info("The file to read does not exist.") return self._underlying_storage.initialize_from_file(file_name=file_name)
def execute_query_file(self, query_file, modification): if not FileUtil.does_file_exist(file_name=query_file): self._SYS_LOGGER.info("Query file " + query_file + " does not exist") query_str = self._query_file_to_str(query_file=query_file) return self.execute_query_str(query_str=query_str, modification=modification)
def read(self, params=None): if self._underlying_storage.get_storage_type( ) == StorageType.PROTO_TABLE_STORAGE: file_base_name = 'data.pb' else: file_base_name = 'data' if params and 'base_name' in params: file_base_name = params['base_name'] params.pop('base_name', None) if params and 'reinitialize_underlying_storage' in params: self._reinitialize_underlying_storage( file_base_name=file_base_name) while self._writer_status != Status.IDLE: self.sys_log("Waiting for writer to finish.") time.sleep(TimeSleepObj.ONE_SECOND) self._reader_status = Status.RUNNING self.sys_log("Read from the latest partition.") latest_dir = self.get_latest_dir() if not latest_dir: self.sys_log("Current partitioner is empty, cannot read anything.") return [] file_name = FileUtil.join_paths_to_file(root_dir=latest_dir, base_name=file_base_name) if not FileUtil.does_file_exist(file_name): self.sys_log("The file [" + file_name + "] to read does not exist.") raise StorageReadException("The file [" + file_name + "] to read does not exist.") if file_name != self._underlying_storage.get_file_name(): self.sys_log("Sync to the latest file to " + file_name) self._underlying_storage.initialize_from_file(file_name=file_name) try: result = self._underlying_storage.read(params=params) self._reader_status = Status.IDLE return result except Exception as err: self.sys_log("Read dir [" + self.get_dir_name() + "] got exception: " + str(err) + '.') self._logger.error("Read dir [" + self.get_dir_name() + "] got exception: " + str(err) + '.') raise StorageReadException("Read dir [" + self.get_dir_name() + "] got exception: " + str(err) + '.')
def make_frontend_config(cls, yaml_path): if not FileUtil.does_file_exist(file_name=yaml_path): return FrontendConfig() else: dict_config = YamlUtil.yaml_to_dict(file_name=yaml_path) config = FrontendConfig() config.container_backend_config.server_url = dict_config[ 'CONTAINER_BACKEND_CONFIG']['SERVER_URL'] config.galaxy_viewer_url = dict_config['GALAXY_VIEWER_URL'] if 'INSTANT_MESSAGING_CONFIG' in dict_config: config.instant_messaging_config.server_url = dict_config[ 'INSTANT_MESSAGING_CONFIG']['SERVER_URL'] if 'EMAIL_CONFIG' in dict_config: config.instant_messaging_config.server_url = dict_config[ 'EMAIL_CONFIG']['SERVER_URL'] credential = Credentials() credential.user_name = dict_config['USER_NAME'] credential.password = dict_config['PASSWORD'] config.credential.CopyFrom(credential) return config