def _has_input(self, input_id: str) -> bool:
input_metadata = InputMetadata.of(
user=self.user,
project=self.project,
path=self.path,
timestamp_millis=self.timestamp_millis)
return self.controller.check_input_data(input_id, input_metadata)
def _put_tarball(self, input_id: str) -> None:
self.tarball.seek(0)
input_metadata = InputMetadata.of(
user=self.user,
project=self.project,
path=self.path,
timestamp_millis=self.timestamp_millis)
self.controller.put_input(input_id=input_id,
input_metadata=input_metadata,
input_data_stream=self.tarball)
def __enter__(self):
# Nothing to save in the context, we have an input id in the controller
# and just refer to it
if self.input_id:
return self
if self.path is None:
raise ValueError('For input data, neither path nor input id were '
'given')
input_metadata = InputMetadata.of(
user=self.user,
project=self.project,
path=self.path,
timestamp_millis=self.timestamp_millis)
# Try to avoid building the tarball. Look at maximum modification
# time in the input, and if we have in input for the timestamp, use
# that one
input_id = self.controller.get_input_id_or_none(input_metadata)
log_debug(f'Input ID from the controller: {input_id}')
if input_id:
log_info('Input files not changed according to modification times')
self.input_id = input_id
return self
log_debug('Building the tarball!')
files = (os.path.join(directory, file)
for directory, _, files in os.walk(self.path)
for file in files)
self.tarball = tempfile.NamedTemporaryFile()
with tarfile.open(self.tarball.name, mode='w:bz2') as tar:
for file in files:
name = os.path.relpath(file, self.path)
size = os.stat(file).st_size
with open(file, 'rb') as f:
tarinfo = tarfile.TarInfo(name=name)
tarinfo.size = size
tar.addfile(tarinfo, fileobj=f)
return self