def setUp(self):
#Set up variables
swisspy_root = swisspy.get_dir_currently_running_in()
self.root = swisspy.smooth_join(swisspy_root, '..')
self.models = swisspy.smooth_join(self.root, 'models')
self.source = swisspy.smooth_join(self.root, 'source')
self.dest = swisspy.smooth_join(self.root, 'dest')
self.logs = swisspy.smooth_join(self.root, 'logs')
self.log_text = log_messages.LogMessage()
self.log_file_path = os.path.join(self.logs, 'test_log_output.txt')
self.input = swisspy.smooth_join(self.models, 'input')
self.input_model = swisspy.smooth_join(self.models,
'enter_paths_model.txt')
self.input_file = swisspy.smooth_join(self.root,
'enter_paths_here.txt')
self.desired_output = swisspy.smooth_join(self.models,
'desired_output')
self.clear_dirs()
# Copy model folder
shutil.copytree(self.input, self.source)
os.mkdir(self.desired_output)
shutil.copy(self.input_model, self.input_file)
os.mkdir(self.dest)
os.mkdir(self.logs)
def set_up_spacer_test(self):
os.mkdir(swisspy.smooth_join(self.source,'spacer'))
os.mkdir(swisspy.smooth_join(self.source,'move_only_from_spacer'))
os.mkdir(swisspy.smooth_join(self.source,
'spacer',
'move_only_from_spacer'))
# Make model directory to compare with
os.mkdir(swisspy.smooth_join(self.desired_output, 'spacer'))
os.mkdir(swisspy.smooth_join(self.desired_output,
'spacer',
'move_only_from_spacer'))
def test_glob_matches_more_than_one_file(self):
test_input = "*/spacer_1"
with open(self.input_file, 'w') as input_file:
input_file.write(test_input)
move_by_regex.move_by_regex(self.source, self.dest, self.input_file)
should_exist = [swisspy.smooth_join(self.dest, 'depth_2', 'spacer_1'),
swisspy.smooth_join(self.dest, 'depth_3', 'spacer_1')]
for dir in should_exist:
self.assertTrue(os.path.exists(dir))
def set_up_spacer_test(self):
os.mkdir(swisspy.smooth_join(self.source, 'spacer'))
os.mkdir(swisspy.smooth_join(self.source, 'move_only_from_spacer'))
os.mkdir(
swisspy.smooth_join(self.source, 'spacer',
'move_only_from_spacer'))
# Make model directory to compare with
os.mkdir(swisspy.smooth_join(self.desired_output, 'spacer'))
os.mkdir(
swisspy.smooth_join(self.desired_output, 'spacer',
'move_only_from_spacer'))
def test_glob_matches_more_than_one_file(self):
test_input = "*/spacer_1"
with open(self.input_file, 'w') as input_file:
input_file.write(test_input)
move_by_regex.move_by_regex(self.source, self.dest, self.input_file)
should_exist = [
swisspy.smooth_join(self.dest, 'depth_2', 'spacer_1'),
swisspy.smooth_join(self.dest, 'depth_3', 'spacer_1')
]
for dir in should_exist:
self.assertTrue(os.path.exists(dir))
def test_move_files_matching_string(self):
goal = swisspy.smooth_join(self.models,
'output_test_move_files_matching_string')
move_by_regex.move_by_regex(self.source, self.dest, self.input_file)
self.assertTrue(swisspy.dirs_match(self.dest, goal))
def test_regex_matching_correctly(self):
test_strings = [
'[pw]eebles', 'l?ummox', '(ben|bill)_and_ben', '4[0-9]{5}',
'boo\\[bar\\]', '([AB])_is_\\1'
]
with open(self.input_file, 'w') as input_file:
for t in test_strings:
input_file.write("container/regex{" + t + "}\n")
container = os.path.join(self.source, 'container')
os.mkdir(container)
# Test directories to put the regex functionality through its paces.
test_dirs = {
'succeed': [
'peebles', 'weebles', 'lummox', 'ummox', 'ben_and_ben',
'bill_and_ben', '412345', 'boo[bar]', 'A_is_A', 'B_is_B'
],
'fail':
['Beebles', 'mummox', 'bill_and_bill', '444', '41234G', 'A_is_B']
}
for t in test_dirs['succeed']:
os.mkdir(os.path.join(container, t))
for t in test_dirs['fail']:
os.mkdir(os.path.join(container, t))
move_by_regex.move_by_regex(self.source, self.dest, self.input_file,
self.log_file_path)
dest_contents = os.listdir(swisspy.smooth_join(self.dest, 'container'))
for s in test_dirs['succeed']:
self.assertIn(s, dest_contents)
for f in test_dirs['fail']:
self.assertNotIn(f, dest_contents)
def test_command_line_flags(self):
command_list = [
os.path.join(self.root, '..', 'move_by_regex.py'),
"-s",
self.source,
"-d",
self.dest,
"-p",
self.input_file,
"-l",
self.log_file_path,
]
goal = swisspy.smooth_join(self.models,
'output_test_move_files_matching_string')
sp.call(command_list)
self.assertTrue(swisspy.dirs_match(self.dest, goal))
# Check log created with read only message
self.assertTrue(os.path.exists(self.log_file_path),
msg=self.log_file_path + " does not exist.")
with open(self.log_file_path, 'r') as log_file:
contents = log_file.read()
self.assertIn(
self.log_text.success_story.format(type="directories",
source=self.source,
dest=self.dest), contents)
def test_correct_behavior_on_no_pattern_file_cmd_line(self):
pattern_file = swisspy.smooth_join(self.root, '..',
'enter_paths_here.txt')
command_list=[
os.path.join(self.root, '..', 'move_by_regex.py'),
"-s", self.source,
"-d", self.dest,
"-l", self.log_file_path,]
expected = self.log_text.no_patterns.format(path_file=pattern_file)
subprocess.call(command_list)
self.assertIn(expected, self.get_log_contents())
def setUp(self):
#Set up variables
swisspy_root = swisspy.get_dir_currently_running_in()
self.root = swisspy.smooth_join(swisspy_root, '..')
self.models = swisspy.smooth_join(self.root, 'models')
self.source = swisspy.smooth_join(self.root, 'source')
self.dest = swisspy.smooth_join(self.root, 'dest')
self.logs = swisspy.smooth_join(self.root, 'logs')
self.log_text = log_messages.LogMessage()
self.log_file_path = os.path.join(self.logs, 'test_log_output.txt')
self.input = swisspy.smooth_join(self.models, 'input')
self.input_model = swisspy.smooth_join(self.models, 'enter_paths_model.txt')
self.input_file = swisspy.smooth_join(self.root, 'enter_paths_here.txt')
self.desired_output = swisspy.smooth_join(self.models, 'desired_output')
self.clear_dirs()
# Copy model folder
shutil.copytree(self.input, self.source)
os.mkdir(self.desired_output)
shutil.copy(self.input_model, self.input_file)
os.mkdir(self.dest)
os.mkdir(self.logs)
def test_correct_behavior_on_no_pattern_file_cmd_line(self):
pattern_file = swisspy.smooth_join(self.root, '..',
'enter_paths_here.txt')
command_list = [
os.path.join(self.root, '..', 'move_by_regex.py'),
"-s",
self.source,
"-d",
self.dest,
"-l",
self.log_file_path,
]
expected = self.log_text.no_patterns.format(path_file=pattern_file)
subprocess.call(command_list)
self.assertIn(expected, self.get_log_contents())
def setUp(self):
# Initialise paths
swisspy_root = swisspy.get_dir_currently_running_in()
self.root = swisspy.swisspy.smooth_join(swisspy_root, '..')
self.models = swisspy.smooth_join(self.root, 'models')
self.source = swisspy.smooth_join(self.root, 'source')
self.logs = swisspy.smooth_join(self.root, 'logs')
self.dest = swisspy.smooth_join(self.root, 'dest')
self.input = swisspy.smooth_join(self.models, 'input')
self.log_file_path = os.path.join(self.logs, 'test_log_output.txt')
self.log_text = log_messages.LogMessage()
self.input_model = swisspy.smooth_join(self.models,
'enter_paths_model.txt')
self.input_file = swisspy.smooth_join(self.models,
'enter_paths_here.txt')
self.log_mod_dest = os.path.join(self.root, 'log_messages.py')
self.test_input = "\nmove_me" \
"\n*/move_me" \
"\nmove_me_too/i_should_also_be_moved" \
"\n# A user comment"
#Clean sweep
self.clear(dirs=[self.source, self.dest, self.logs],
files=[self.input_file])
# Copy model folder
try:
shutil.rmtree(self.source)
except OSError as e:
error_number = e[0]
if error_number == 2: #File doesn't exist
pass
else:
raise
shutil.copytree(self.input, self.source)
os.mkdir(self.dest)
os.mkdir(self.logs)
# Copy and populate input file
shutil.copy(self.input_model, self.input_file)
with open(self.input_file, 'a') as input_file:
input_file.write(self.test_input)
def test_files_within_matched_dirs_not_found_by_search_source(self):
self.set_up_spacer_test()
a_file_path = (swisspy.smooth_join(self.source, 'spacer',
'move_only_from_spacer', 'a_file'))
with open(a_file_path, 'w') as a_file:
a_file.write("DO NOT FIND ME")
operation = move_by_regex.search_source_for_patterns
observed = operation(self.source,
[['spacer', 'move_only_from_spacer'],
['spacer', 'move_only_from_spacer', 'a_file']])
desired = {
'dirs_to_move':
[os.path.join(self.source, 'spacer', 'move_only_from_spacer')],
'files_to_move': [],
'invalid_regex': [],
'paths_matched': ['spacer/move_only_from_spacer'],
'paths_not_matched': [],
'redundant_paths': ['spacer/move_only_from_spacer/a_file']
}
self.assertEqual(observed, desired)
def test_files_within_matched_dirs_not_found_by_search_source(self):
self.set_up_spacer_test()
a_file_path = (swisspy.smooth_join(self.source, 'spacer',
'move_only_from_spacer',
'a_file'))
with open(a_file_path, 'w') as a_file:
a_file.write("DO NOT FIND ME")
operation = move_by_regex.search_source_for_patterns
observed = operation(self.source,
[['spacer','move_only_from_spacer'],
['spacer', 'move_only_from_spacer', 'a_file']])
desired = {'dirs_to_move':[os.path.join(self.source,
'spacer',
'move_only_from_spacer')],
'files_to_move':[],
'invalid_regex':[],
'paths_matched':['spacer/move_only_from_spacer'],
'paths_not_matched':[],
'redundant_paths':['spacer/move_only_from_spacer/a_file']}
self.assertEqual(observed,desired)
def search_source_for_patterns(source, patterns,
regex_ind_start=None, regex_ind_end=None):
"""
Walk the source directory and return a list of paths which match patterns.
This is the meat. If anything's gone awry, it's probably this function.
source : str : path
The source directory to search for patterns
patterns : list : lists
A list of patterns
regex_ind_start : str
String indicating the beginning of a regex pattern
regex_ind_end : str
String indicating the end of a regex pattern
:return dict
{'dirs_to_move' : list of dirs to move,
'files_to_move' : list of files to move,
'paths_matched' : list of paths queried and found in source
'paths_not_matched' : list of paths queried but not found in source
'redundant_paths' : list of paths not searched as a parent dir had
already been found}
'invalid_regex' : Invalid regex patterns encountered.
"""
dirs_to_move = []
files_to_move=[]
satisfied = []
invalid_regex = []
# Remove any redundant patterns before going on (e.g ['usr','bin'] is
# redundant if ['usr'] is present.
redundant_patterns_output = get_redundant_patterns(patterns)
redundant_patterns = redundant_patterns_output['redundant']
to_check = redundant_patterns_output['not_redundant']
for root, dirs, files in os.walk(source):
walk_depth = path_depth_difference(root, source)
# Since we remove matches as we go, only to_check representing paths
# which are deeper than we've walked so far are eligible to match.
possible_patterns = [p for p in to_check if len(p) - 1 >= walk_depth]
if not possible_patterns:
continue
for pattern in possible_patterns:
pieces = [PatternPiece(p) for p in pattern]
to_match = pieces[walk_depth]
for d in dirs[:]:
try:
paths_match = match(to_match, d)
except Exception:
if to_match.regex_pattern not in invalid_regex:
invalid_regex.append(to_match.regex_pattern)
to_check.remove(pattern)
continue
if paths_match:
# We've got a match! If that's the end of the pattern,
# move this object
if len(pattern) - 1 == walk_depth:
dir_path = swisspy.smooth_join(root, d)
dirs_to_move.append(dir_path)
# Remove this dir from dirs to be walked
dirs.remove(d)
satisfied.append(pattern)
# Only remove this from the list to check if it doesn't
# contain a glob
non_strings = [q for q in pieces if q.type != 'string']
if not non_strings:
to_check.remove(pattern)
for f in files:
try:
paths_match = match(to_match, f)
except Exception:
if to_match.regex_pattern not in invalid_regex:
invalid_regex.append(to_match.regex_pattern)
to_check.remove(pattern)
continue
if paths_match:
if len(pattern) - 1 == walk_depth:
file_path = swisspy.smooth_join(root, f)
files_to_move.append(file_path)
files.remove(f)
satisfied.append(pattern)
non_strings = [q for q in pieces if q.type != 'string']
if not non_strings:
to_check.remove(pattern)
sep = os.path.sep
paths_not_matched = [sep.join(t) for t in to_check if t not in satisfied]
paths_matched = [sep.join(s) for s in satisfied]
redundant_paths = [sep.join(r) for r in redundant_patterns]
out_dict = {'dirs_to_move':dirs_to_move,
'files_to_move':files_to_move,
'invalid_regex':invalid_regex,
'paths_matched':paths_matched,
'paths_not_matched':paths_not_matched,
'redundant_paths':redundant_paths,}
return out_dict
def main():
swisspy_path = swisspy.get_dir_currently_running_in()
current_dir = swisspy.smooth_join(swisspy_path, '..')
args = init_args(current_dir)
move_by_regex(args.source, args.dest, args.paths_file, args.log_file,
args.read_only, args.log_unmatched)
def move_by_regex(source, dest, paths_file="", log_file="", read_only=False,
log_unmatched=False):
# Set up variables
dir_successes = []
file_successes = []
log_text = log_messages.LogMessage()
# Set up logging
init_logging(log_file, log_text)
main_logger = logging.getLogger('mbr.main')
swisspy_path = swisspy.get_dir_currently_running_in()
current_dir = swisspy.smooth_join(swisspy_path, '..')
if not paths_file:
paths_file = os.path.join(current_dir, 'enter_paths_here.txt')
if not log_file:
log_file = os.path.join(current_dir, 'logs', 'move_log.txt')
paths = get_lines(paths_file)
main_logger.info("\n".join(paths))
if paths:
patterns = get_patterns(paths)
search_result = search_source_for_patterns(source, patterns)
if read_only:
if search_result['dirs_to_move']:
header = log_text.found_files_header.format(type='Directories')
main_logger.info(header)
main_logger.info('\n\t' +\
'\n\t'.join(search_result['dirs_to_move']))
if search_result['files_to_move']:
header = log_text.found_files_header.format(type='Files')
main_logger.info(header)
main_logger.info('\n\t' +\
'\n\t'.join(search_result['files_to_move']))
else:
for dir_path in search_result['dirs_to_move']:
dir_successes = move_creating_intermediaries(source,
dir_path,
dest)
if dir_successes:
header = log_text.success_story.format(type='directories',
source=source,
dest=dest)
main_logger.info(header)
for ds in dir_successes:
main_logger.info("\t" + join_pattern(ds))
for file_path in search_result['files_to_move']:
file_successes = move_creating_intermediaries(source,
file_path,
dest)
if file_successes:
header = log_text.success_story.format(type='files',
source=source,
dest=dest)
main_logger.info(header)
for fs in file_successes:
main_logger.info("\t" + join_pattern(fs))
if log_unmatched:
main_logger.info(log_text.unmatched_header)
for p in search_result['paths_not_matched']:
main_logger.info(p)
else:
main_logger.info(log_text.no_patterns.format(path_file=paths_file))