def save(self, result):
result_file_path = self._get_result_file_path(result)
try:
make_parent_dirs_if_nonexist(result_file_path)
except OSError as e:
print 'make_parent_dirs_if_nonexist {path} fails: {e}'.format(
path=result_file_path, e=str(e))
pass
with open(result_file_path, "wt") as result_file:
result_file.write(str(result.to_dataframe().to_dict()))
def _prepare_log_file(self, asset):
log_file_path = self._get_log_file_path(asset)
# if parent dir doesn't exist, create
make_parent_dirs_if_nonexist(log_file_path)
# add runner type and version
with open(log_file_path, 'wt') as log_file:
log_file.write("{type_version_str}\n\n".format(
type_version_str=self.get_cozy_type_version_string()))
def save(self, result):
result_file_path = self._get_result_file_path(result)
try:
make_parent_dirs_if_nonexist(result_file_path)
except OSError as e:
print 'make_parent_dirs_if_nonexist {path} fails: {e}'.format(
path=result_file_path,
e=str(e))
pass
with open(result_file_path, "wt") as result_file:
result_file.write(str(result.to_dataframe().to_dict()))
def _prepare_log_file(self, asset):
log_file_path = self._get_log_file_path(asset)
# if parent dir doesn't exist, create
make_parent_dirs_if_nonexist(log_file_path)
# add runner type and version
with open(log_file_path, 'wt') as log_file:
log_file.write("{type_version_str}\n\n".format(
type_version_str=self.get_cozy_type_version_string()))
def _run_on_asset(self, asset):
# Override Executor._run_on_asset to skip working on ref video
if self.result_store:
result = self.result_store.load(asset, self.executor_id)
else:
result = None
# if result can be retrieved from result_store, skip log file
# generation and reading result from log file, but directly return
# return the retrieved result
if result is not None:
if self.logger:
self.logger.info('{id} result exists. Skip {id} run.'.
format(id=self.executor_id))
else:
if self.logger:
self.logger.info('{id} result does\'t exist. Perform {id} '
'calculation.'.format(id=self.executor_id))
# at this stage, it is certain that asset.ref_path and
# asset.dis_path will be used. must early determine that
# they exists
self._assert_paths(asset)
# if no rescaling is involved, directly work on ref_path/dis_path,
# instead of opening workfiles
self._set_asset_use_path_as_workpath(asset)
# remove workfiles if exist (do early here to avoid race condition
# when ref path and dis path have some overlap)
if asset.use_path_as_workpath:
# do nothing
pass
else:
self._close_dis_workfile(asset)
log_file_path = self._get_log_file_path(asset)
make_parent_dirs_if_nonexist(log_file_path)
if asset.use_path_as_workpath:
# do nothing
pass
else:
if self.fifo_mode:
dis_p = multiprocessing.Process(target=self._open_dis_workfile,
args=(asset, True))
dis_p.start()
self._wait_for_workfiles(asset)
else:
self._open_dis_workfile(asset, fifo_mode=False)
self._prepare_log_file(asset)
self._generate_result(asset)
# clean up workfiles
if self.delete_workdir:
if asset.use_path_as_workpath:
# do nothing
pass
else:
self._close_dis_workfile(asset)
if self.logger:
self.logger.info("Read {id} log file, get scores...".
format(type=self.executor_id))
# collect result from each asset's log file
result = self._read_result(asset)
# save result
if self.result_store:
self.result_store.save(result)
# clean up workdir and log files in it
if self.delete_workdir:
# remove log file
self._remove_log(asset)
# remove dir
log_file_path = self._get_log_file_path(asset)
log_dir = get_dir_without_last_slash(log_file_path)
try:
os.rmdir(log_dir)
except OSError as e:
if e.errno == 39: # [Errno 39] Directory not empty
# VQM could generate an error file with non-critical
# information like: '3 File is longer than 15 seconds.
# Results will be calculated using first 15 seconds
# only.' In this case, want to keep this
# informational file and pass
pass
result = self._post_process_result(result)
return result
def save(self, result):
result_file_path = self._get_result_file_path(result)
make_parent_dirs_if_nonexist(result_file_path)
with open(result_file_path, "wt") as result_file:
result_file.write(str(result.to_dataframe().to_dict()))
def save(self, result):
result_file_path = self._get_result_file_path(result)
make_parent_dirs_if_nonexist(result_file_path)
with open(result_file_path, "wt") as result_file:
result_file.write(str(result.to_dataframe().to_dict()))
def _run_on_asset(self, asset):
# Wraper around the essential function _generate_result, to
# do housekeeping work including 1) asserts of asset, 2) skip run if
# log already exist, 3) creating fifo, 4) delete work file and dir
# asserts
self._assert_an_asset(asset)
if self.result_store:
result = self.result_store.load(asset, self.executor_id)
else:
result = None
# if result can be retrieved from result_store, skip log file
# generation and reading result from log file, but directly return
# return the retrieved result
if result is not None:
if self.logger:
self.logger.info('{id} result exists. Skip {id} run.'.
format(id=self.executor_id))
else:
if self.logger:
self.logger.info('{id} result does\'t exist. Perform {id} '
'calculation.'.format(id=self.executor_id))
# at this stage, it is certain that asset.ref_path and
# asset.dis_path will be used. must early determine that
# they exists
self._assert_paths(asset)
# if no rescaling is involved, directly work on ref_path/dis_path,
# instead of opening workfiles
self._set_asset_use_path_as_workpath(asset)
# remove workfiles if exist (do early here to avoid race condition
# when ref path and dis path have some overlap)
if asset.use_path_as_workpath:
# do nothing
pass
else:
self._close_ref_workfile(asset)
self._close_dis_workfile(asset)
log_file_path = self._get_log_file_path(asset)
make_parent_dirs_if_nonexist(log_file_path)
if asset.use_path_as_workpath:
# do nothing
pass
else:
if self.fifo_mode:
ref_p = multiprocessing.Process(target=self._open_ref_workfile,
args=(asset, True))
dis_p = multiprocessing.Process(target=self._open_dis_workfile,
args=(asset, True))
ref_p.start()
dis_p.start()
self._wait_for_workfiles(asset)
else:
self._open_ref_workfile(asset, fifo_mode=False)
self._open_dis_workfile(asset, fifo_mode=False)
self._prepare_log_file(asset)
self._generate_result(asset)
# clean up workfiles
if self.delete_workdir:
if asset.use_path_as_workpath:
# do nothing
pass
else:
self._close_ref_workfile(asset)
self._close_dis_workfile(asset)
if self.logger:
self.logger.info("Read {id} log file, get scores...".
format(type=self.executor_id))
# collect result from each asset's log file
result = self._read_result(asset)
# save result
if self.result_store:
self.result_store.save(result)
# clean up workdir and log files in it
if self.delete_workdir:
# remove log file
self._remove_log(asset)
# remove dir
log_file_path = self._get_log_file_path(asset)
log_dir = get_dir_without_last_slash(log_file_path)
try:
os.rmdir(log_dir)
except OSError as e:
if e.errno == 39: # [Errno 39] Directory not empty
# VQM could generate an error file with non-critical
# information like: '3 File is longer than 15 seconds.
# Results will be calculated using first 15 seconds
# only.' In this case, want to keep this
# informational file and pass
pass
result = self._post_process_result(result)
return result