def _create_directory(dirpath: pathlib.Path): """ Create local directory for saving object results before starting the object. Attributes: dirpath: `pathlib.Path` directory path. """ logger.info( '[LocalRunner] Creating a local directory for saving object ' f'results: {dirpath}') created = create_local_directory(dirpath)
def _create_directory(dirpath: pathlib.Path): """ Create a local directory for saving experiment results. Raises: SrtUtilsException """ logger.info( '[SingleExperimentRunner] Creating a local directory for saving ' f'experiment results: {dirpath}') created = create_local_directory(dirpath) if not created: raise SrtUtilsException( 'Directory for saving experiment results already exists: ' f'{dirpath}. Please use non-existing directory name and ' 'start the experiment again. Existing directory contents ' 'will not be deleted')
def collect_results(self): """ Before collecting object results, this function creates a local directory `username@host` inside self.collect_results_path directory where the results produced by the object are copied. """ logger.info(f'Collecting object results: {self.obj}, {self.process}') before_collect_results_checks(self.obj, self.process, self.collect_results_path) # If an object has filepath equal to None, it means there should be # no output file produced if self.obj.filepath is None: logger.info( 'There was no output file expected, nothing to collect') return # If an object has filepath defined, it means there should be # an output file produced. However it does not mean that the file # was created successfully, that's why we check whether the filepath exists. with fabric.Connection(host=self.host, user=self.username) as c: if not exists(c, self.obj.filepath): stdout, stderr = self.process.collect_results() raise SrtUtilsException( 'There was no output file produced by the object: ' f'{self.obj}, nothing to collect. Process stdout: ' f'{stdout}. Process stderr: {stderr}') # Create 'username@host' folder to copy produced by the object file # (inside self.collect_results_path directory) destination_dir = self.collect_results_path / f'{self.username}@{self.host}' logger.info('Creating a local directory for copying object results: ' f'{destination_dir}') created = create_local_directory(destination_dir) # if not created: # logger.info( # 'Directory already exists, no need to create: ' # f'{destination_dir}' # ) logger.info(f'Copying object results into: {destination_dir}') filename = self.obj.filepath.name source = self.obj.filepath destination = destination_dir / filename if destination.exists(): raise SrtUtilsException( 'The destination file already exists, there might be a ' f'file created by the other object: {destination}. File ' f'with object results was not copied: {self.obj.filepath}') # TODO: Implement copying files using rsync try: # http://docs.fabfile.org/en/2.3/api/transfer.html with fabric.Connection(host=self.host, user=self.username) as c: result = c.get(source, destination) except OSError as error: raise SrtUtilsException( f'Object results have not been collected: {self.obj.filepath}' f'. Exception occured ({error.__class__.__name__}): {error}. ') except Exception as error: logger.info('Most probably paramiko exception') raise SrtUtilsException( f'Object results have not been collected: {self.obj.filepath}' f'. Exception occured ({error.__class__.__name__}): {error}. ')
def collect_results(self): """ Before collecting object results, this function creates a local directory `local` inside self.collect_results_path directory where the results produced by the object are copied. """ logger.info(f'Collecting object results: {self.obj}, {self.process}') before_collect_results_checks(self.obj, self.process, self.collect_results_path) # If an object has filepath equal to None, it means there should be # no output file produced if self.obj.filepath is None: logger.info( 'There was no output file expected, nothing to collect') return # If an object has filepath defined, it means there should be # an output file produced. However it does not mean that the file # was created successfully, that's why we check whether the filepath exists. if not self.obj.filepath.exists(): stdout, stderr = self.process.collect_results() raise SrtUtilsException( 'There was no output file produced by the object: ' f'{self.obj}, nothing to collect. Process stdout: ' f'{stdout}. Process stderr: {stderr}') # Create 'local' folder to copy produced by the object file # (inside self.collect_results_path directory) destination_dir = self.collect_results_path / 'local' logger.info('Creating a local directory for copying object results: ' f'{destination_dir}') created = create_local_directory(destination_dir) # if not created: # logger.info( # 'Directory already exists, no need to create: ' # f'{destination_dir}' # ) # The code below will raise a FileExistsError if destination already exists. # Technically, this copies a file. To perform a move, simply delete source # after the copy is done. Make sure no exception was raised though. # In case we have several tasks which is runned locally by # LocalRunner runner and in case the tasks have the same names # for the output files, the result might be overwritten. # That's why we do not delete destination file before, instead # we catch FileExistsError exception. That's why it is necessary # to make sure that the file names for different tasks are unique. logger.info(f'Copying object results into: {destination_dir}') filename = self.obj.filepath.name source = self.obj.filepath destination = destination_dir / filename try: with destination.open(mode='xb') as fid: fid.write(source.read_bytes()) except FileExistsError: raise SrtUtilsException( 'The destination file already exists, there might be a ' f'file created by the other object: {destination}. File ' f'with object results was not copied: {self.obj.filepath}')