def _create_directory(dirpath: pathlib.Path):
"""
Create local directory for saving object results before
starting the object.
Attributes:
dirpath:
`pathlib.Path` directory path.
"""
logger.info(
'[LocalRunner] Creating a local directory for saving object '
f'results: {dirpath}')
created = create_local_directory(dirpath)
def _create_directory(dirpath: pathlib.Path):
"""
Create a local directory for saving experiment results.
Raises:
SrtUtilsException
"""
logger.info(
'[SingleExperimentRunner] Creating a local directory for saving '
f'experiment results: {dirpath}')
created = create_local_directory(dirpath)
if not created:
raise SrtUtilsException(
'Directory for saving experiment results already exists: '
f'{dirpath}. Please use non-existing directory name and '
'start the experiment again. Existing directory contents '
'will not be deleted')
def collect_results(self):
"""
Before collecting object results, this function creates a local
directory `username@host` inside self.collect_results_path directory
where the results produced by the object are copied.
"""
logger.info(f'Collecting object results: {self.obj}, {self.process}')
before_collect_results_checks(self.obj, self.process,
self.collect_results_path)
# If an object has filepath equal to None, it means there should be
# no output file produced
if self.obj.filepath is None:
logger.info(
'There was no output file expected, nothing to collect')
return
# If an object has filepath defined, it means there should be
# an output file produced. However it does not mean that the file
# was created successfully, that's why we check whether the filepath exists.
with fabric.Connection(host=self.host, user=self.username) as c:
if not exists(c, self.obj.filepath):
stdout, stderr = self.process.collect_results()
raise SrtUtilsException(
'There was no output file produced by the object: '
f'{self.obj}, nothing to collect. Process stdout: '
f'{stdout}. Process stderr: {stderr}')
# Create 'username@host' folder to copy produced by the object file
# (inside self.collect_results_path directory)
destination_dir = self.collect_results_path / f'{self.username}@{self.host}'
logger.info('Creating a local directory for copying object results: '
f'{destination_dir}')
created = create_local_directory(destination_dir)
# if not created:
# logger.info(
# 'Directory already exists, no need to create: '
# f'{destination_dir}'
# )
logger.info(f'Copying object results into: {destination_dir}')
filename = self.obj.filepath.name
source = self.obj.filepath
destination = destination_dir / filename
if destination.exists():
raise SrtUtilsException(
'The destination file already exists, there might be a '
f'file created by the other object: {destination}. File '
f'with object results was not copied: {self.obj.filepath}')
# TODO: Implement copying files using rsync
try:
# http://docs.fabfile.org/en/2.3/api/transfer.html
with fabric.Connection(host=self.host, user=self.username) as c:
result = c.get(source, destination)
except OSError as error:
raise SrtUtilsException(
f'Object results have not been collected: {self.obj.filepath}'
f'. Exception occured ({error.__class__.__name__}): {error}. ')
except Exception as error:
logger.info('Most probably paramiko exception')
raise SrtUtilsException(
f'Object results have not been collected: {self.obj.filepath}'
f'. Exception occured ({error.__class__.__name__}): {error}. ')
def collect_results(self):
"""
Before collecting object results, this function creates a local
directory `local` inside self.collect_results_path directory
where the results produced by the object are copied.
"""
logger.info(f'Collecting object results: {self.obj}, {self.process}')
before_collect_results_checks(self.obj, self.process,
self.collect_results_path)
# If an object has filepath equal to None, it means there should be
# no output file produced
if self.obj.filepath is None:
logger.info(
'There was no output file expected, nothing to collect')
return
# If an object has filepath defined, it means there should be
# an output file produced. However it does not mean that the file
# was created successfully, that's why we check whether the filepath exists.
if not self.obj.filepath.exists():
stdout, stderr = self.process.collect_results()
raise SrtUtilsException(
'There was no output file produced by the object: '
f'{self.obj}, nothing to collect. Process stdout: '
f'{stdout}. Process stderr: {stderr}')
# Create 'local' folder to copy produced by the object file
# (inside self.collect_results_path directory)
destination_dir = self.collect_results_path / 'local'
logger.info('Creating a local directory for copying object results: '
f'{destination_dir}')
created = create_local_directory(destination_dir)
# if not created:
# logger.info(
# 'Directory already exists, no need to create: '
# f'{destination_dir}'
# )
# The code below will raise a FileExistsError if destination already exists.
# Technically, this copies a file. To perform a move, simply delete source
# after the copy is done. Make sure no exception was raised though.
# In case we have several tasks which is runned locally by
# LocalRunner runner and in case the tasks have the same names
# for the output files, the result might be overwritten.
# That's why we do not delete destination file before, instead
# we catch FileExistsError exception. That's why it is necessary
# to make sure that the file names for different tasks are unique.
logger.info(f'Copying object results into: {destination_dir}')
filename = self.obj.filepath.name
source = self.obj.filepath
destination = destination_dir / filename
try:
with destination.open(mode='xb') as fid:
fid.write(source.read_bytes())
except FileExistsError:
raise SrtUtilsException(
'The destination file already exists, there might be a '
f'file created by the other object: {destination}. File '
f'with object results was not copied: {self.obj.filepath}')