def move_files_to_copies(self):
with self._storage.create_session(read_only=False,
locked=True) as session:
files_with_hashes = session\
.query(File.relative_path, File.file_hash) \
.filter(File.is_folder == 0) \
.all()
copies_dir = get_copies_dir(self._root)
for (file, hashsum) in files_with_hashes:
hash_path = op.join(copies_dir, hashsum)
file_path = self._path_converter.create_abspath(file)
if not op.exists(hash_path):
try:
os.rename(file_path, hash_path)
except Exception as e:
logger.error("Error moving file to copy: %s", e)
remove_file(file_path)
abs_path = FilePath(self._root).longpath
folders_plus_hidden = [
self._path_converter.create_abspath(f)
for f in os.listdir(abs_path) if f not in HIDDEN_DIRS
]
for folder in folders_plus_hidden:
if not op.isdir(folder):
continue
try:
remove_dir(folder)
except Exception as e:
logger.error("Error removing dir '%s' (%s)", folder, e)
logger.info("Removed all files and folders")
self._storage.clean()
def _clean_spec_files(self):
logger.debug("Cleaning spec files from '%s'...", self._filename)
try:
with open(self._filename, 'rb') as f:
self._spec_files = pickle.load(f)
for share_hash in self._spec_files:
path, is_directory = self._spec_files[share_hash]
try:
if is_directory:
remove_dir(path)
else:
remove_file(path)
logger.debug("Special file (folder) removed %s", path)
except Exception as e:
logger.warning("Can't delete file or folder %s. Reason %s",
path, e)
except Exception as e:
logger.warning("Failed to load special files data (%s)", e)
try:
remove_file(self._filename)
except Exception:
pass
self._spec_files = dict()
try:
with open(self._filename, 'wb') as f:
pickle.dump(self._spec_files, f, protocol=2)
except Exception as e:
logger.warning("Failed to save special files data (%s)", e)
def _delete_empty_file_or_folder(self):
self._is_deleting = True
fullname = self._get_full_name(cancel=False, existing_file=True)
if fullname:
self._fullname = fullname
if not self._fullname:
self._is_deleting = False
return
self._fullname = FilePath(self._fullname)
logger.debug("Removing special file %s", self._fullname)
if not self._in_data_dir:
self._sync.remove_special_file(self._fullname)
try:
if self._is_folder:
remove_dir(self._fullname)
else:
remove_file(self._fullname)
except Exception as e:
logger.warning("Can't delete file or folder %s. Reason %s",
self._fullname, e)
self._update_spec_files()
if self._in_data_dir:
self._sync.remove_special_file(self._fullname)
self._fullname = ""
self._is_deleting = False
def _clear_share_download(self):
data_dir = self._cfg.sync_directory if self._cfg else get_data_dir()
downloads_dir = get_downloads_dir(data_dir=data_dir, create=True)
download_name = op.join(downloads_dir, self._current_share_hash)
if self._is_folder:
remove_dir(download_name)
else:
remove_file(download_name)
def save(self):
utils.remove_dir(self.path)
utils.make_dir(self.path)
foo = lambda model, params, name: save_model(
model,
os.path.join(self.path, name + MODEL_STATE_EXT),
params
)
self._apply(foo, self.model_names + ['stacker'])
def _init_temp_dir(self):
self._temp_dir = get_temp_dir(self._root)
if exists(self._temp_dir):
try:
remove_dir(self._temp_dir)
except Exception as e:
logger.warning("Can't remove temp dir. Reason: %s", e)
self._temp_dir = get_temp_dir(self._root, create=True)
def clean(self):
files = self._storage.get_known_files()
for file in files:
try:
remove_file(file)
except Exception as e:
logger.error("Error removing file '%s' (%s)", file, e)
folders = self._storage.get_known_folders()
for folder in sorted(folders, key=len):
try:
remove_dir(folder)
except Exception as e:
logger.error("Error removing dir '%s' (%s)", folder, e)
logger.info("Removed all files and folders")
self._storage.clean()
def delete_directory(self, full_path, events_file_id=None):
full_path = unicodedata.normalize('NFC', full_path)
with self._storage.create_session(read_only=False,
locked=True) as session:
file, _full_path = self._get_file_by_id(events_file_id, session)
if file:
full_path = _full_path
elif events_file_id is not None:
logger.warning("Skipping directory deletion because "
"directory with same events_file_id not found")
return
rel_path = self._path_converter.create_relpath(full_path)
files = self._storage.get_known_folder_children(rel_path,
session=session)
try:
temp_path = join(self._temp_dir, basename(full_path))
if isdir(temp_path):
remove_dir(temp_path, suppress_not_exists_exception=True)
elif isfile(temp_path):
remove_file(temp_path)
if isdir(full_path):
os.rename(full_path, temp_path)
try:
remove_dir(temp_path,
suppress_not_exists_exception=True)
except Exception:
logger.debug("Dir %s delete failed", temp_path)
except OSError as e:
logger.warning("Can't remove dir %s. Reason: %s", full_path, e)
if e.errno == errno.EACCES:
self._raise_access_denied(full_path)
elif e.errno != errno.ENOENT: # directory does not exist
raise e
deleted_paths = [f.relative_path for f in files]
self._storage.delete_known_folder_children(rel_path,
session=session)
for path in deleted_paths:
self.file_deleted.emit(path)
def prepare_data_in_folders(self, x, y, path):
"""
It copies images into their corresponding class folders.
:param x: list of images paths
:param y: list of classes/labels
:param path: base path where class folders will be created
"""
full_path = os.path.abspath(path)
if os.path.exists(full_path):
# just in case it exists and contains previous executions data
remove_dir(full_path)
else:
create_dir(full_path)
i = 0
for f in x:
fname = f.split('/')[-1]
cls = get_value(y[i], self.classes)
dst = os.path.join(full_path, cls)
if not os.path.exists(dst):
create_dir(dst)
dst = os.path.join(dst, fname)
copy_file(f, dst)
i += 1
def _delete(self, dirs=[], files=[]):
for dir in dirs:
remove_dir(dir)
for file in files:
remove_file(file)