def choose_project_path():
self.project_path = filedialog.askdirectory()
biba = 1
clear_folder(self.project_path)
# files = os.listdir(self.project_path)
# for file in files:
# file_path = os.path.join(self.project_path, file)
# if os.path.isfile(file_path) or os.path.islink(file_path):
# os.unlink(file_path)
# elif os.path.isdir(file_path):
# shutil.rmtree(file_path)
b_choose_project_path.destroy()
self.create_main_buttons()
def test_walk_tree_with_rule(self):
clear_folder(self.directory_path)
prepare_filetree(self.directory_path, '''
ignore/b/c:1
ignore/bb:2
include/d/e: 3
include/dd: 4
whatever_file: 5
''')
rule_set = FileRuleSet()
rule_set.add_rule_by_string("ignore: name eq ignore")
filetree = FileTree.from_fs(self.directory_path, rule_set=rule_set)
self.assertEqual(len(filetree.files()), 3)
self.assertEqual(len(filetree.folders()), 2)
def test_walk_tree_with_rule(self):
clear_folder(self.directory_path)
prepare_filetree(
self.directory_path, '''
ignore/b/c:1
ignore/bb:2
include/d/e: 3
include/dd: 4
whatever_file: 5
''')
rule_set = FileRuleSet()
rule_set.add_rule_by_string("ignore: name eq ignore")
filetree = FileTree.from_fs(self.directory_path, rule_set=rule_set)
self.assertEqual(len(filetree.files()), 3)
self.assertEqual(len(filetree.folders()), 2)
def process(self, do_clear_info=True):
logging.info("starting tree processing")
start = u.timestamp_now()
u.ensure_path(self.config.output)
self._root_files_processed = 0
if self.clear_first and do_clear_info:
u.clear_folder(self.config.output)
# do this at start in case last run didn't clean up properly
self.remove_unpacked_files()
if do_clear_info:
self.file_info.clear()
self._pass = 0 # pass number
self._files_processed = 0
# make one pass over the input files. if you need to know whether this is
# the input pass, check for self._pass == 0.
self._walk_files(self.config.input)
if self.config.signalled():
logging.info("signal set, leaving tp.process")
return False
# then make passes over the output files until no new files are encountered
work_done = self._files_processed > 0
Config.log('tp._files_processed = %i' % self._files_processed, tag='WORK_DONE_PASS_0')
# do NOT look at _root_files_processed after pass 0 - we want to fully
# process any files created during pass 0
while self._pass < self.PASSES:
self._files_processed = 0
self._pass += 1
self._walk_files(self.config.output)
if self.config.signalled():
logging.info("signal set, leaving tp.process after pass %i" % self._pass)
work_done = False
break
Config.log('tp._files_processed = %i' % self._files_processed, tag='WORK_DONE_PASS_%i' % self._pass)
if self._files_processed > 0:
work_done = True
else:
break
if self._pass >= self.PASSES:
raise Exception("completed %i passes and still not done. failing" % self.PASSES)
self.update_input_mgr_metadata()
elapsed = u.timestamp_now() - start
Config.log("tp completed in %i passes, %f seconds, work_done %s" % (self._pass, elapsed, work_done), tag='WORK_DONE')
return work_done
def clear_folders(self):
clear_folder(self.plain_folder)
clear_folder(self.plain_folder_check)
clear_folder(self.encrypted_folder)
def sync_to_upstream_dest_mgr(self, upstream, refresh_me, refresh_upstream, tmp_folder, fixer=None):
msg = "refresh_me = %s, refresh_upstream = %s, upstream root = '%s', my root = '%s', tmp_folder = %s" % \
(refresh_me, refresh_upstream, upstream._file_dest_root, self._file_dest_root, tmp_folder)
Config.log(msg, tag='FILE_DEST_SYNC_TO_UPSTREAM_STARTING')
do_refresh = {'refresh_dest_meta': True}
dont_refresh = {'refresh_dest_meta': False}
smart_refresh = {'refresh_dest_meta': True, 'skip_refresh_if_tree_unchanged': True}
if refresh_me == 'full':
self.sync_tree_info(options=do_refresh)
elif refresh_me == 'smart':
self.sync_tree_info(options=smart_refresh)
else:
self.sync_tree_info(options=dont_refresh)
if refresh_upstream == 'full':
upstream.sync_tree_info(options=do_refresh)
elif refresh_upstream == 'smart':
upstream.sync_tree_info(options=smart_refresh)
else:
upstream.sync_tree_info(options=dont_refresh)
u.clear_folder(tmp_folder)
start = u.timestamp_now()
for key, finfo in upstream.tree_info_items():
src = os.path.join(upstream._file_dest_root, key)
if not os.path.exists(src):
msg = "file '%s' does not exist" % src
Config.log(msg, tag='FILE_DEST_SYNC_TO_UPSTREAM_METADATA ERROR')
continue
if self.config.is_template_type(finfo['name']) and fixer:
# copy and fix up
dest = os.path.join(tmp_folder, key)
u.ensure_path_for_file(dest)
shutil.copyfile(src, dest)
fixer(dest)
# make new metadata and copy to self
newfi = copy.deepcopy(finfo)
path, file = os.path.split(dest)
local_meta = u.local_metadata(path, file)
newfi['size'] = local_meta['size']
newfi['modified'] = local_meta['modified']
newfi['md5'] = u.md5(dest)
self.tree_info[key] = newfi
src = dest
dest = os.path.join(self._file_dest_root, key)
u.ensure_path_for_file(dest)
shutil.copyfile(src, dest)
# we could remove fixed-up file now, but clear_folder at end probably faster
else:
# file not subject to fixup. just copy if missing/older/diff size
copyit = False
if key in self.tree_info:
# compare metadata and see whether to copy
myfi = self.tree_info[key]
if myfi['md5'] != finfo['md5'] or \
myfi['modified'] < finfo['modified'] or \
myfi['size'] != finfo['size']:
copyit = True
else:
copyit = True
if copyit:
# REVIEW - deepcopy probably safe here because we're copying from
# one dest mgr to another
self.tree_info[key] = copy.deepcopy(finfo)
dest = os.path.join(self._file_dest_root, key)
u.ensure_path_for_file(dest)
shutil.copyfile(src, dest)
# delete from me if not in upstream
to_delete = {}
for key, finfo in self.tree_info_items():
if key not in upstream.tree_info:
to_delete[key] = os.path.join(self._file_dest_root, finfo['key'])
for key, full in to_delete.items():
os.remove(full)
del self.tree_info[key]
self.write_tree_info()
# this is a space-saving move, but should be small, and might
# be handy to have files around for debug. could be a config option.
# u.clear_folder(tmp_folder)
elapsed = u.timestamp_now() - start
msg = "done, elapsed %f seconds" % elapsed
Config.log(msg, tag='FILE_DEST_SYNC_TO_UPSTREAM_FINISHED')
def clear_folders(self):
clear_folder(self.plain_folder)
clear_folder(self.plain_folder_check)
clear_folder(self.encrypted_folder)
def remove_unpacked_files(self):
u.clear_folder(self.unpack_root)
