def add_sqlite_file(ctx: BackupContext,
tar: TarFile,
src: t.Union[os.PathLike, str],
arcname: str) -> None:
if not path.isfile(src):
return
with NamedTemporaryFile(mode='rb', suffix='.db') as db_backup_file:
ctx.logger.debug('Dumping database from %s to %s',
src, db_backup_file.name)
subprocess.run(
['sqlite3', str(src), f".backup '{db_backup_file.name}'"],
check=True)
# 元ファイルのメタデータを使用する
src_stat = os.stat(src)
ti = tar.gettarinfo(db_backup_file.name, arcname)
ti.mtime = src_stat.st_mtime
ti.mode = src_stat.st_mode
ti.uid = src_stat.st_uid
ti.gid = src_stat.st_gid
ti.uname = ''
ti.gname = ''
with open(src, 'rb') as f:
tar.addfile(ti, f)
def save_to_file(self, f):
tar = TarFile(f, "w")
# save info file
f = StringIO(repr((self.agedesc, self.generation)))
info = tar.gettarinfo(None, "info.py", f)
tar.addfile(info, f)
f.close()
# save agents
for i in range(len(self.agents)):
f = StringIO()
self.agents[i].save_to_file(f)
info = tar.gettarinfo(None, str(i)+".agt", f)
tar.addfile(info, f)
f.close()
tar.close()
def save_to_file(self, f):
tar = TarFile(f, "w")
# save info file
f = StringIO(repr((self.agedesc, self.generation)))
info = tar.gettarinfo(None, "info.py", f)
tar.addfile(info, f)
f.close()
# save agents
for i in range(len(self.agents)):
f = StringIO()
self.agents[i].save_to_file(f)
info = tar.gettarinfo(None, str(i) + ".agt", f)
tar.addfile(info, f)
f.close()
tar.close()
def _add_pyproject(self, tar: tarfile.TarFile, tar_dir: str) -> None:
"""Rewrites the pyproject.toml before adding to tarball.
This is mainly aiming at fixing the version number in pyproject.toml
"""
pyproject = toml.loads(self.meta.filepath.read_text("utf-8"))
if not isinstance(self.meta._metadata.get("version", ""), str):
self.meta._metadata["version"] = self.meta.version
pyproject["project"] = self.meta._metadata
name = self.meta.filepath.name
tarinfo = tar.gettarinfo(name, os.path.join(tar_dir, name))
bio = io.BytesIO(toml.dumps(pyproject).encode("utf-8"))
tarinfo.size = len(bio.getvalue())
tar.addfile(tarinfo, bio)
def add_file(tar: tarfile.TarFile,
file_name: str) -> Tuple[int, int, datetime, Optional[str]]:
# FIXME: error: "TarFile" has no attribute "offset"
offset: int = tar.offset # type: ignore
tarinfo: tarfile.TarInfo = tar.gettarinfo(file_name)
# Change the size of any hardlinks from 0 to the size of the actual file
if tarinfo.islnk():
tarinfo.size = os.path.getsize(file_name)
# Add the file to the tar
tar.addfile(tarinfo)
md5: Optional[str] = None
# Only add files or hardlinks.
# (So don't add directories or softlinks.)
if tarinfo.isfile() or tarinfo.islnk():
f: _io.TextIOWrapper = open(file_name, "rb")
hash_md5: _hashlib.HASH = hashlib.md5()
if tar.fileobj is not None:
fileobj: _io.BufferedWriter = tar.fileobj
else:
raise TypeError("Invalid tar.fileobj={}".format(tar.fileobj))
while True:
s: str = f.read(BLOCK_SIZE)
if len(s) > 0:
# If the block read in is non-empty, write it to fileobj and update the hash
fileobj.write(s)
hash_md5.update(s)
if len(s) < BLOCK_SIZE:
# If the block read in is smaller than BLOCK_SIZE,
# then we have reached the end of the file.
# blocks = how many blocks of tarfile.BLOCKSIZE fit in tarinfo.size
# remainder = how much more content is required to reach tarinfo.size
blocks: int
remainder: int
blocks, remainder = divmod(tarinfo.size, tarfile.BLOCKSIZE)
if remainder > 0:
null_bytes: bytes = tarfile.NUL
# Write null_bytes to get the last block to tarfile.BLOCKSIZE
fileobj.write(null_bytes * (tarfile.BLOCKSIZE - remainder))
blocks += 1
# Increase the offset by the amount already saved to the tar
# FIXME: error: "TarFile" has no attribute "offset"
tar.offset += blocks * tarfile.BLOCKSIZE # type: ignore
break
f.close()
md5 = hash_md5.hexdigest()
size: int = tarinfo.size
mtime: datetime = datetime.utcfromtimestamp(tarinfo.mtime)
return offset, size, mtime, md5
def _add_pyproject(self, tar: tarfile.TarFile, tar_dir: str) -> None:
"""Rewrites the pyproject.toml before adding to tarball.
This is mainly aiming at fixing the version number in pyproject.toml
"""
with self.meta.filepath.open("rb") as f:
pyproject = tomli.load(f)
if self.meta.dynamic and "version" in self.meta.dynamic:
self.meta._metadata["version"] = self.meta.version
self.meta._metadata["dynamic"].remove("version")
pyproject["project"] = self.meta._metadata
name = self.meta.filepath.name
tarinfo = tar.gettarinfo(name, os.path.join(tar_dir, name))
bio = io.BytesIO()
tomli_w.dump(pyproject, bio)
tarinfo.size = len(bio.getvalue())
bio.seek(0)
tar.addfile(tarinfo, bio)
def _add_pyproject(self, tar: tarfile.TarFile, tar_dir: str) -> None:
"""Rewrites the pyproject.toml before adding to tarball.
This is mainly aiming at fixing the version number in pyproject.toml
"""
pyproject_content = self._meta.filepath.read_text()
if not isinstance(self._meta._metadata.get("version", ""), str):
pyproject_content = re.sub(
r"^version *= *.+?$",
f'version = "{self._meta.version}"',
pyproject_content,
flags=re.M,
)
name = "pyproject.toml"
tarinfo = tar.gettarinfo(name, os.path.join(tar_dir, name))
bio = io.BytesIO(pyproject_content.encode("utf-8"))
tarinfo.size = len(bio.getvalue())
tar.addfile(tarinfo, bio)
def export_to_tar(self, tarfile: TarFile, destination_dir: str, mtime: int = BST_ARBITRARY_TIMESTAMP) -> None:
# We need directories here, including non-empty ones,
# so list_relative_paths is not used.
for filename in sorted(os.listdir(self.__external_directory)):
source_name = os.path.join(self.__external_directory, filename)
arcname = os.path.join(destination_dir, filename)
tarinfo = tarfile.gettarinfo(source_name, arcname)
tarinfo.mtime = mtime
tarinfo.uid = 0
tarinfo.gid = 0
tarinfo.uname = ""
tarinfo.gname = ""
if tarinfo.isreg():
with open(source_name, "rb") as f:
tarfile.addfile(tarinfo, f)
elif tarinfo.isdir():
tarfile.addfile(tarinfo)
self.open_directory(filename).export_to_tar(tarfile, arcname, mtime)
else:
tarfile.addfile(tarinfo)
