def test_os_filesystem(self):
if is_travis:
pytest.skip(
"skipping windows filesystem test since travis has no NTFS",
allow_module_level=True,
)
if os_name == "nt":
# assume there is a c:\ on the test machine (which is likely)
do_check(Path("C:\\"))
self.assertFalse(get_check().is_linux)
else:
do_check(test_data)
self.assertTrue(get_check().is_linux)
def fs_checks(root_folder: Path, args: dict):
Utils.minimum_date(root_folder)
# store the root folder filesystem checks globally for all to inspect
do_check(root_folder, int(args.max_filename), bool(args.ntfs))
# check if symlinks are supported
if not get_check().is_symlink:
args.skip_albums = True
# check if file system is case sensitive
if not args.case_insensitive_fs:
if not get_check().is_case_sensitive:
args.case_insensitive_fs = True
return args
def test_fs_overrides(self):
s = ts.SetupDbAndCredentials()
args = ["--ntfs", "--max-filename", "30"]
s.test_setup("test_fs_overrides",
args=args,
trash_db=True,
trash_files=True)
s.gp.fs_checks(s.root, s.parsed_args)
self.assertFalse(get_check().is_linux)
self.assertEquals(get_check().max_filename, 30)
if os_name != "nt":
args = []
s.test_setup("test_fs_overrides",
args=args,
trash_db=True,
trash_files=True)
s.gp.fs_checks(s.root, s.parsed_args)
self.assertTrue(get_check().is_linux)
self.assertEquals(get_check().max_filename, 255)
def orig_name(self) -> Path:
try:
name = self.__media_json["filename"]
matches = DuplicateSuffix.match(name)
if matches:
# append the prefix and the suffix, ditching the ' (n)'
name = "{}{}".format(*matches.groups())
except KeyError:
name = ""
if self.__lower:
name = name.lower()
return Path(get_check().valid_file_name(name))
def album_folder_name(self, album_name: str, start_date: datetime,
end_date: datetime) -> Path:
album_name = get_check().valid_file_name(album_name)
if self._omit_album_date:
rel_path = album_name
else:
if self.use_start_date:
d = start_date
else:
d = end_date
year = Utils.safe_str_time(d, "%Y")
month = Utils.safe_str_time(d, "%m%d")
if self._use_flat_path:
rel_path = "{0}-{1} {2}".format(year, month, album_name)
else:
rel_path = Path(year) / "{0} {1}".format(month, album_name)
link_folder: Path = self._links_root / rel_path
return link_folder
def create_album_content_links(self):
log.warning("Creating album folder links to media ...")
count = 0
album_item = 0
current_rid = ""
# always re-create all album links - it is quite fast and a good way
# to ensure consistency
# especially now that we have --album-date-by-first-photo
if self._links_root.exists():
log.debug("removing previous album links tree")
shutil.rmtree(self._links_root)
re_download = not self._links_root.exists()
for (
path,
file_name,
album_name,
start_date_str,
end_date_str,
rid,
created,
) in self._db.get_album_files(download_again=re_download):
if current_rid == rid:
album_item += 1
else:
self._db.put_album_downloaded(rid)
current_rid = rid
album_item = 0
end_date = Utils.string_to_date(end_date_str)
start_date = Utils.string_to_date(start_date_str)
if len(str(self._root_folder / path)) > get_check().max_path:
max_path_len = get_check().max_path - len(
str(self._root_folder))
log.debug("This path needs to be shrunk: %s" %
Path(self._root_folder / path))
path = path[:max_path_len]
log.debug("Shrunk to: %s" % Path(self._root_folder / path))
file_name = file_name[:get_check().max_filename]
full_file_name = self._root_folder / path / file_name
link_folder: Path = self.album_folder_name(album_name, start_date,
end_date)
link_file = link_folder / "{:04d}_{}".format(album_item, file_name)
# incredibly, pathlib.Path.relative_to cannot handle
# '../' in a relative path !!! reverting to os.path
relative_filename = os.path.relpath(full_file_name,
str(link_folder))
log.debug("adding album link %s -> %s", relative_filename,
link_file)
try:
if not link_folder.is_dir():
log.debug("new album folder %s", link_folder)
link_folder.mkdir(parents=True)
created_date = Utils.string_to_date(created)
if full_file_name.exists():
if self._use_hardlinks:
os.link(full_file_name, link_file)
elif self._ntfs_override:
os.symlink(relative_filename, link_file)
else:
link_file.symlink_to(relative_filename)
else:
log.debug("skip link for %s, not downloaded", file_name)
if link_file.exists():
count += 1
# Windows tries to follow symlinks even though we specify
# follow_symlinks=False. So disable setting of link date
# if follow not supported
try:
if os.utime in os.supports_follow_symlinks:
os.utime(
str(link_file),
(
Utils.safe_timestamp(
created_date).timestamp(),
Utils.safe_timestamp(
created_date).timestamp(),
),
follow_symlinks=False,
)
except PermissionError:
log.debug(f"cant set date on {link_file}")
except (FileExistsError, UnicodeEncodeError):
log.error("bad link to %s", full_file_name)
log.warning("Created %d new album folder links", count)
def description(self) -> str:
try:
return get_check().valid_file_name(
self.__media_json["description"])
except KeyError:
return ""
def orig_name(self) -> str:
"""
Original filename before duplicate name handling
"""
return get_check().valid_file_name(self._orig_name)
def description(self) -> str:
"""
The description of the file
"""
return get_check().valid_file_name(self._description)
def filename(self) -> str:
"""
filename including a suffix to make it unique if duplicates exist
"""
return get_check().valid_file_name(self._filename)
