def save(self, result):
result_file_path = self._get_result_file_path(result)
try:
make_parent_dirs_if_nonexist(result_file_path)
except OSError as e:
print('make_parent_dirs_if_nonexist {path} fails: {e}'.format(
path=result_file_path, e=str(e)))
self.save_result(result, result_file_path)
def _prepare_log_file(self, asset):
log_file_path = self._get_log_file_path(asset)
# if parent dir doesn't exist, create
make_parent_dirs_if_nonexist(log_file_path)
# add runner type and version
with open(log_file_path, 'wt') as log_file:
log_file.write("{type_version_str}\n\n".format(
type_version_str=self.get_cozy_type_version_string()))
def save(self, result):
result_file_path = self._get_result_file_path(result)
try:
make_parent_dirs_if_nonexist(result_file_path)
except OSError as e:
print 'make_parent_dirs_if_nonexist {path} fails: {e}'.format(
path=result_file_path,
e=str(e))
pass
self.save_result(result, result_file_path)
def _prepare_log_file(self, asset):
log_file_path = self._get_log_file_path(asset)
# if parent dir doesn't exist, create
make_parent_dirs_if_nonexist(log_file_path)
# add runner type and version
with open(log_file_path, 'wt') as log_file:
log_file.write("{type_version_str}\n\n".format(
type_version_str=self.get_cozy_type_version_string()))
def _run_on_asset(self, asset):
# Wraper around the essential function _generate_result, to
# do housekeeping work including 1) asserts of asset, 2) skip run if
# log already exist, 3) creating fifo, 4) delete work file and dir
if self.result_store:
result = self.result_store.load(asset, self.executor_id)
else:
result = None
# if result can be retrieved from result_store, skip log file
# generation and reading result from log file, but directly return
# return the retrieved result
if result is not None:
if self.logger:
self.logger.info('{id} result exists. Skip {id} run.'.format(
id=self.executor_id))
else:
if self.logger:
self.logger.info('{id} result does\'t exist. Perform {id} '
'calculation.'.format(id=self.executor_id))
# at this stage, it is certain that asset.ref_path and
# asset.dis_path will be used. must early determine that
# they exists
self._assert_paths(asset)
# if no rescaling is involved, directly work on ref_path/dis_path,
# instead of opening workfiles
self._set_asset_use_path_as_workpath(asset)
# remove workfiles if exist (do early here to avoid race condition
# when ref path and dis path have some overlap)
if asset.use_path_as_workpath:
# do nothing
pass
else:
self._close_workfiles(asset)
log_file_path = self._get_log_file_path(asset)
make_parent_dirs_if_nonexist(log_file_path)
if asset.use_path_as_workpath:
# do nothing
pass
else:
if self.fifo_mode:
self._open_workfiles_in_fifo_mode(asset)
else:
self.open_workfiles(asset)
self._prepare_log_file(asset)
self._generate_result(asset)
# clean up workfiles
if self.delete_workdir:
if asset.use_path_as_workpath:
# do nothing
pass
else:
self._close_workfiles(asset)
if self.logger:
self.logger.info("Read {id} log file, get scores...".format(
id=self.executor_id))
# collect result from each asset's log file
result = self._read_result(asset)
# save result
if self.result_store:
result = self._save_result(result)
# clean up workdir and log files in it
if self.delete_workdir:
# remove log file
self._remove_log(asset)
# remove dir
log_file_path = self._get_log_file_path(asset)
log_dir = get_dir_without_last_slash(log_file_path)
try:
os.rmdir(log_dir)
except OSError as e:
if e.errno == 39: # [Errno 39] Directory not empty
# e.g. VQM could generate an error file with non-critical
# information like: '3 File is longer than 15 seconds.
# Results will be calculated using first 15 seconds
# only.' In this case, want to keep this
# informational file and pass
pass
result = self._post_process_result(result)
return result
def _run_on_asset(self, asset):
# Wraper around the essential function _generate_result, to
# do housekeeping work including 1) asserts of asset, 2) skip run if
# log already exist, 3) creating fifo, 4) delete work file and dir
if self.result_store:
result = self.result_store.load(asset, self.executor_id)
else:
result = None
# if result can be retrieved from result_store, skip log file
# generation and reading result from log file, but directly return
# return the retrieved result
if result is not None:
if self.logger:
self.logger.info('{id} result exists. Skip {id} run.'.
format(id=self.executor_id))
else:
if self.logger:
self.logger.info('{id} result does\'t exist. Perform {id} '
'calculation.'.format(id=self.executor_id))
# at this stage, it is certain that asset.ref_path and
# asset.dis_path will be used. must early determine that
# they exists
self._assert_paths(asset)
# if no rescaling is involved, directly work on ref_path/dis_path,
# instead of opening workfiles
self._set_asset_use_path_as_workpath(asset)
# remove workfiles if exist (do early here to avoid race condition
# when ref path and dis path have some overlap)
if asset.use_path_as_workpath:
# do nothing
pass
else:
self._close_workfiles(asset)
log_file_path = self._get_log_file_path(asset)
make_parent_dirs_if_nonexist(log_file_path)
if asset.use_path_as_workpath:
# do nothing
pass
else:
if self.fifo_mode:
self._open_workfiles_in_fifo_mode(asset)
else:
self.open_workfiles(asset)
self._prepare_log_file(asset)
self._generate_result(asset)
# clean up workfiles
if self.delete_workdir:
if asset.use_path_as_workpath:
# do nothing
pass
else:
self._close_workfiles(asset)
if self.logger:
self.logger.info("Read {id} log file, get scores...".
format(type=self.executor_id))
# collect result from each asset's log file
result = self._read_result(asset)
# save result
if self.result_store:
result = self._save_result(result)
# clean up workdir and log files in it
if self.delete_workdir:
# remove log file
self._remove_log(asset)
# remove dir
log_file_path = self._get_log_file_path(asset)
log_dir = get_dir_without_last_slash(log_file_path)
try:
os.rmdir(log_dir)
except OSError as e:
if e.errno == 39: # [Errno 39] Directory not empty
# e.g. VQM could generate an error file with non-critical
# information like: '3 File is longer than 15 seconds.
# Results will be calculated using first 15 seconds
# only.' In this case, want to keep this
# informational file and pass
pass
result = self._post_process_result(result)
return result
