def _reinitialize_underlying_storage(self, file_base_name):
file_name = FileUtil.join_paths_to_file(root_dir=self.get_latest_dir(),
base_name=file_base_name)
if not FileUtil.does_file_exist(file_name):
self.sys_log("The file to read does not exist.")
return
self._underlying_storage.initialize_from_file(file_name=file_name)
def _reinitialize_underlying_storage(self, file_base_name):
file_name = FileUtil.join_paths_to_file(
root_dir=self._get_latest_dir_internal(), base_name=file_base_name)
self.increment_rpc_count_by(n=1)
if not FileUtil.does_file_exist(file_name):
self._SYS_LOGGER.info("The file to read does not exist.")
return
self._underlying_storage.initialize_from_file(file_name=file_name)
def execute_query_file(self, query_file, modification):
if not FileUtil.does_file_exist(file_name=query_file):
self._SYS_LOGGER.info("Query file " + query_file +
" does not exist")
query_str = self._query_file_to_str(query_file=query_file)
return self.execute_query_str(query_str=query_str,
modification=modification)
def read(self, params=None):
if self._underlying_storage.get_storage_type(
) == StorageType.PROTO_TABLE_STORAGE:
file_base_name = 'data.pb'
else:
file_base_name = 'data'
if params and 'base_name' in params:
file_base_name = params['base_name']
params.pop('base_name', None)
if params and 'reinitialize_underlying_storage' in params:
self._reinitialize_underlying_storage(
file_base_name=file_base_name)
while self._writer_status != Status.IDLE:
self.sys_log("Waiting for writer to finish.")
time.sleep(TimeSleepObj.ONE_SECOND)
self._reader_status = Status.RUNNING
self.sys_log("Read from the latest partition.")
latest_dir = self.get_latest_dir()
if not latest_dir:
self.sys_log("Current partitioner is empty, cannot read anything.")
return []
file_name = FileUtil.join_paths_to_file(root_dir=latest_dir,
base_name=file_base_name)
if not FileUtil.does_file_exist(file_name):
self.sys_log("The file [" + file_name +
"] to read does not exist.")
raise StorageReadException("The file [" + file_name +
"] to read does not exist.")
if file_name != self._underlying_storage.get_file_name():
self.sys_log("Sync to the latest file to " + file_name)
self._underlying_storage.initialize_from_file(file_name=file_name)
try:
result = self._underlying_storage.read(params=params)
self._reader_status = Status.IDLE
return result
except Exception as err:
self.sys_log("Read dir [" + self.get_dir_name() +
"] got exception: " + str(err) + '.')
self._logger.error("Read dir [" + self.get_dir_name() +
"] got exception: " + str(err) + '.')
raise StorageReadException("Read dir [" + self.get_dir_name() +
"] got exception: " + str(err) + '.')
def make_frontend_config(cls, yaml_path):
if not FileUtil.does_file_exist(file_name=yaml_path):
return FrontendConfig()
else:
dict_config = YamlUtil.yaml_to_dict(file_name=yaml_path)
config = FrontendConfig()
config.container_backend_config.server_url = dict_config[
'CONTAINER_BACKEND_CONFIG']['SERVER_URL']
config.galaxy_viewer_url = dict_config['GALAXY_VIEWER_URL']
if 'INSTANT_MESSAGING_CONFIG' in dict_config:
config.instant_messaging_config.server_url = dict_config[
'INSTANT_MESSAGING_CONFIG']['SERVER_URL']
if 'EMAIL_CONFIG' in dict_config:
config.instant_messaging_config.server_url = dict_config[
'EMAIL_CONFIG']['SERVER_URL']
credential = Credentials()
credential.user_name = dict_config['USER_NAME']
credential.password = dict_config['PASSWORD']
config.credential.CopyFrom(credential)
return config
