def choose_project_path(): self.project_path = filedialog.askdirectory() biba = 1 clear_folder(self.project_path) # files = os.listdir(self.project_path) # for file in files: # file_path = os.path.join(self.project_path, file) # if os.path.isfile(file_path) or os.path.islink(file_path): # os.unlink(file_path) # elif os.path.isdir(file_path): # shutil.rmtree(file_path) b_choose_project_path.destroy() self.create_main_buttons()
def test_walk_tree_with_rule(self): clear_folder(self.directory_path) prepare_filetree(self.directory_path, ''' ignore/b/c:1 ignore/bb:2 include/d/e: 3 include/dd: 4 whatever_file: 5 ''') rule_set = FileRuleSet() rule_set.add_rule_by_string("ignore: name eq ignore") filetree = FileTree.from_fs(self.directory_path, rule_set=rule_set) self.assertEqual(len(filetree.files()), 3) self.assertEqual(len(filetree.folders()), 2)
def test_walk_tree_with_rule(self): clear_folder(self.directory_path) prepare_filetree( self.directory_path, ''' ignore/b/c:1 ignore/bb:2 include/d/e: 3 include/dd: 4 whatever_file: 5 ''') rule_set = FileRuleSet() rule_set.add_rule_by_string("ignore: name eq ignore") filetree = FileTree.from_fs(self.directory_path, rule_set=rule_set) self.assertEqual(len(filetree.files()), 3) self.assertEqual(len(filetree.folders()), 2)
def process(self, do_clear_info=True): logging.info("starting tree processing") start = u.timestamp_now() u.ensure_path(self.config.output) self._root_files_processed = 0 if self.clear_first and do_clear_info: u.clear_folder(self.config.output) # do this at start in case last run didn't clean up properly self.remove_unpacked_files() if do_clear_info: self.file_info.clear() self._pass = 0 # pass number self._files_processed = 0 # make one pass over the input files. if you need to know whether this is # the input pass, check for self._pass == 0. self._walk_files(self.config.input) if self.config.signalled(): logging.info("signal set, leaving tp.process") return False # then make passes over the output files until no new files are encountered work_done = self._files_processed > 0 Config.log('tp._files_processed = %i' % self._files_processed, tag='WORK_DONE_PASS_0') # do NOT look at _root_files_processed after pass 0 - we want to fully # process any files created during pass 0 while self._pass < self.PASSES: self._files_processed = 0 self._pass += 1 self._walk_files(self.config.output) if self.config.signalled(): logging.info("signal set, leaving tp.process after pass %i" % self._pass) work_done = False break Config.log('tp._files_processed = %i' % self._files_processed, tag='WORK_DONE_PASS_%i' % self._pass) if self._files_processed > 0: work_done = True else: break if self._pass >= self.PASSES: raise Exception("completed %i passes and still not done. failing" % self.PASSES) self.update_input_mgr_metadata() elapsed = u.timestamp_now() - start Config.log("tp completed in %i passes, %f seconds, work_done %s" % (self._pass, elapsed, work_done), tag='WORK_DONE') return work_done
def clear_folders(self): clear_folder(self.plain_folder) clear_folder(self.plain_folder_check) clear_folder(self.encrypted_folder)
def sync_to_upstream_dest_mgr(self, upstream, refresh_me, refresh_upstream, tmp_folder, fixer=None): msg = "refresh_me = %s, refresh_upstream = %s, upstream root = '%s', my root = '%s', tmp_folder = %s" % \ (refresh_me, refresh_upstream, upstream._file_dest_root, self._file_dest_root, tmp_folder) Config.log(msg, tag='FILE_DEST_SYNC_TO_UPSTREAM_STARTING') do_refresh = {'refresh_dest_meta': True} dont_refresh = {'refresh_dest_meta': False} smart_refresh = {'refresh_dest_meta': True, 'skip_refresh_if_tree_unchanged': True} if refresh_me == 'full': self.sync_tree_info(options=do_refresh) elif refresh_me == 'smart': self.sync_tree_info(options=smart_refresh) else: self.sync_tree_info(options=dont_refresh) if refresh_upstream == 'full': upstream.sync_tree_info(options=do_refresh) elif refresh_upstream == 'smart': upstream.sync_tree_info(options=smart_refresh) else: upstream.sync_tree_info(options=dont_refresh) u.clear_folder(tmp_folder) start = u.timestamp_now() for key, finfo in upstream.tree_info_items(): src = os.path.join(upstream._file_dest_root, key) if not os.path.exists(src): msg = "file '%s' does not exist" % src Config.log(msg, tag='FILE_DEST_SYNC_TO_UPSTREAM_METADATA ERROR') continue if self.config.is_template_type(finfo['name']) and fixer: # copy and fix up dest = os.path.join(tmp_folder, key) u.ensure_path_for_file(dest) shutil.copyfile(src, dest) fixer(dest) # make new metadata and copy to self newfi = copy.deepcopy(finfo) path, file = os.path.split(dest) local_meta = u.local_metadata(path, file) newfi['size'] = local_meta['size'] newfi['modified'] = local_meta['modified'] newfi['md5'] = u.md5(dest) self.tree_info[key] = newfi src = dest dest = os.path.join(self._file_dest_root, key) u.ensure_path_for_file(dest) shutil.copyfile(src, dest) # we could remove fixed-up file now, but clear_folder at end probably faster else: # file not subject to fixup. just copy if missing/older/diff size copyit = False if key in self.tree_info: # compare metadata and see whether to copy myfi = self.tree_info[key] if myfi['md5'] != finfo['md5'] or \ myfi['modified'] < finfo['modified'] or \ myfi['size'] != finfo['size']: copyit = True else: copyit = True if copyit: # REVIEW - deepcopy probably safe here because we're copying from # one dest mgr to another self.tree_info[key] = copy.deepcopy(finfo) dest = os.path.join(self._file_dest_root, key) u.ensure_path_for_file(dest) shutil.copyfile(src, dest) # delete from me if not in upstream to_delete = {} for key, finfo in self.tree_info_items(): if key not in upstream.tree_info: to_delete[key] = os.path.join(self._file_dest_root, finfo['key']) for key, full in to_delete.items(): os.remove(full) del self.tree_info[key] self.write_tree_info() # this is a space-saving move, but should be small, and might # be handy to have files around for debug. could be a config option. # u.clear_folder(tmp_folder) elapsed = u.timestamp_now() - start msg = "done, elapsed %f seconds" % elapsed Config.log(msg, tag='FILE_DEST_SYNC_TO_UPSTREAM_FINISHED')
def remove_unpacked_files(self): u.clear_folder(self.unpack_root)