def _has_input(self, input_id: str) -> bool: input_metadata = InputMetadata.of( user=self.user, project=self.project, path=self.path, timestamp_millis=self.timestamp_millis) return self.controller.check_input_data(input_id, input_metadata)
def _put_tarball(self, input_id: str) -> None: self.tarball.seek(0) input_metadata = InputMetadata.of( user=self.user, project=self.project, path=self.path, timestamp_millis=self.timestamp_millis) self.controller.put_input(input_id=input_id, input_metadata=input_metadata, input_data_stream=self.tarball)
def __enter__(self): # Nothing to save in the context, we have an input id in the controller # and just refer to it if self.input_id: return self if self.path is None: raise ValueError('For input data, neither path nor input id were ' 'given') input_metadata = InputMetadata.of( user=self.user, project=self.project, path=self.path, timestamp_millis=self.timestamp_millis) # Try to avoid building the tarball. Look at maximum modification # time in the input, and if we have in input for the timestamp, use # that one input_id = self.controller.get_input_id_or_none(input_metadata) log_debug(f'Input ID from the controller: {input_id}') if input_id: log_info('Input files not changed according to modification times') self.input_id = input_id return self log_debug('Building the tarball!') files = (os.path.join(directory, file) for directory, _, files in os.walk(self.path) for file in files) self.tarball = tempfile.NamedTemporaryFile() with tarfile.open(self.tarball.name, mode='w:bz2') as tar: for file in files: name = os.path.relpath(file, self.path) size = os.stat(file).st_size with open(file, 'rb') as f: tarinfo = tarfile.TarInfo(name=name) tarinfo.size = size tar.addfile(tarinfo, fileobj=f) return self